Compare commits
488
Commits
ccc6c742ee
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fa7750046 | ||
|
|
1838eed855 | ||
|
|
96d8a5e3f0 | ||
|
|
0431e720de | ||
|
|
7f4921de49 | ||
|
|
d4c19baa32 | ||
|
|
d444fd8036 | ||
|
|
d42b1e2dda | ||
|
|
9492dc4c39 | ||
|
|
172beca3c5 | ||
|
|
f0c1289519 | ||
|
|
128172d3a8 | ||
|
|
820e8325a6 | ||
|
|
74c6a0f5eb | ||
|
|
d289a9101c | ||
|
|
9553ecb16a | ||
|
|
fb104bf05a | ||
|
|
f4adc31215 | ||
|
|
c6254f2342 | ||
|
|
d25b147a56 | ||
|
|
1f9915f074 | ||
|
|
c0e7a97a56 | ||
|
|
68c835f33b | ||
|
|
6b9fda76db | ||
|
|
1be259b66d | ||
|
|
2c3c88d067 | ||
|
|
3fc14d16bf | ||
|
|
4cc50889d6 | ||
|
|
63e68740b6 | ||
|
|
f2fddeb87d | ||
|
|
eda411c0be | ||
|
|
492757ce70 | ||
|
|
b58282b3d6 | ||
|
|
9da2760a42 | ||
|
|
049f633667 | ||
|
|
84946ed0c6 | ||
|
|
514e13660c | ||
|
|
9333334b7b | ||
|
|
367d0203b8 | ||
|
|
37017882c7 | ||
|
|
f116a1584e | ||
|
|
1a5bacce87 | ||
|
|
f6e5f7dcd6 | ||
|
|
0f0c407458 | ||
|
|
354992c9e7 | ||
|
|
28987240aa | ||
|
|
fbf2b92167 | ||
|
|
206a119a4b | ||
|
|
671e7ea5a4 | ||
|
|
d5cf3db2ec | ||
|
|
dc8823724d | ||
|
|
a392108562 | ||
|
|
fd1ab58598 | ||
|
|
bd13bdc6e1 | ||
|
|
bdf1bce4be | ||
|
|
c5c0477653 | ||
|
|
76faf0f9f8 | ||
|
|
879109e55d | ||
|
|
61b7d2a96a | ||
|
|
15d802ae3d | ||
|
|
ac17be2cd9 | ||
|
|
f1ba418aa8 | ||
|
|
b17cc3a09d | ||
|
|
1c38abe1d0 | ||
|
|
5d42a28d34 | ||
|
|
e5f4ede3f2 | ||
|
|
ce613bcf9d | ||
|
|
8ce8f0dfbe | ||
|
|
13c221244e | ||
|
|
a89b704fa1 | ||
|
|
0f228020ef | ||
|
|
8454d3d0a2 | ||
|
|
da15cc7531 | ||
|
|
5924aae1c0 | ||
|
|
46239012b4 | ||
|
|
f8119f3786 | ||
|
|
41c0598220 | ||
|
|
387382ec9f | ||
|
|
20998c8c31 | ||
|
|
f5d6edd9d0 | ||
|
|
4808a57289 | ||
|
|
96b907acbc | ||
|
|
d9c37ac763 | ||
|
|
146be2d0ad | ||
|
|
906375dc23 | ||
|
|
e1cdf03d40 | ||
|
|
8041da5424 | ||
|
|
7c4f881b57 | ||
|
|
81abc5adee | ||
|
|
331b7b6b13 | ||
|
|
fe23c59e0c | ||
|
|
666e1ebab2 | ||
|
|
3d05276312 | ||
|
|
00fdee5649 | ||
|
|
4a29e7bc99 | ||
|
|
a07019e3aa | ||
|
|
aa16fbaa8b | ||
|
|
9c15df8e22 | ||
|
|
111c70d6b4 | ||
|
|
0621f66889 | ||
|
|
4accd6b67e | ||
|
|
63ec110177 | ||
|
|
a27bb41b42 | ||
|
|
eaa78452bc | ||
|
|
4eb0a9c608 | ||
|
|
2e646f82b4 | ||
|
|
74bd40ba78 | ||
|
|
4f99efcf2c | ||
|
|
be10e78c7a | ||
|
|
b6a677a710 | ||
|
|
53d7784d5d | ||
|
|
10f2a0a8a4 | ||
|
|
eab5cc911c | ||
|
|
d88fe381db | ||
|
|
ecded4a8a7 | ||
|
|
772a3bdce2 | ||
|
|
fbf6a07da9 | ||
|
|
25e055a7ac | ||
|
|
e024a29157 | ||
|
|
83f3b7e226 | ||
|
|
d3d6f30f72 | ||
|
|
99c94ffaec | ||
|
|
1a836dac75 | ||
|
|
0447fa6f86 | ||
|
|
e8ee5b052b | ||
|
|
d7980da35e | ||
|
|
239e100b44 | ||
|
|
2216072669 | ||
|
|
8cf3ff31db | ||
|
|
b108a6fe62 | ||
|
|
f58ff68ab3 | ||
|
|
e7e9d9fecf | ||
|
|
d417bd5360 | ||
|
|
90f3b0b7da | ||
|
|
bce3bf61e9 | ||
|
|
49372996ac | ||
|
|
e3ed5a53f2 | ||
|
|
1231cd69a8 | ||
|
|
8e85a67e7b | ||
|
|
1badb93331 | ||
|
|
e23358ea0c | ||
|
|
73bbae14c1 | ||
|
|
74e3b8b067 | ||
|
|
9a1254048e | ||
|
|
7ae510ed28 | ||
|
|
620d7eb6f1 | ||
|
|
cd21fca749 | ||
|
|
ead5d38d7c | ||
|
|
c4bfa6bdfe | ||
|
|
d8a77c700b | ||
|
|
5fc53948c6 | ||
|
|
587440b6bc | ||
|
|
fb49eb20e8 | ||
|
|
60e87ba518 | ||
|
|
ff94ccb343 | ||
|
|
42711ff9ac | ||
|
|
7c957d96be | ||
|
|
427a31cdff | ||
|
|
51125fa05a | ||
|
|
24e347bb2d | ||
|
|
1a47f864f9 | ||
|
|
ad623353bc | ||
|
|
9d2610a911 | ||
|
|
bc9692f521 | ||
|
|
d8e3c09b57 | ||
|
|
7e7d845f61 | ||
|
|
b83c118c7f | ||
|
|
069815790b | ||
|
|
366e2a269f | ||
|
|
bdc2cdbadb | ||
|
|
fe58fb7826 | ||
|
|
6dbb076a1e | ||
|
|
3676526daf | ||
|
|
6a15a9d99a | ||
|
|
a46317de18 | ||
|
|
32355e0219 | ||
|
|
2cbc06a683 | ||
|
|
cce1e25c2b | ||
|
|
39dba3f8ab | ||
|
|
5bdf3bff60 | ||
|
|
65516ea3ac | ||
|
|
27989d066e | ||
|
|
3ae6298656 | ||
|
|
662f4f0d53 | ||
|
|
a02b918ff9 | ||
|
|
23825a824d | ||
|
|
cf67393db6 | ||
|
|
7ed0f5c23b | ||
|
|
f363c6cd55 | ||
|
|
c8a6f58007 | ||
|
|
93cdbaac57 | ||
|
|
47ad3b3075 | ||
|
|
1493cf2dcf | ||
|
|
35e59d2510 | ||
|
|
4f95cc6d13 | ||
|
|
f627658567 | ||
|
|
3c24550b19 | ||
|
|
98fab394a1 | ||
|
|
6fbc2b4125 | ||
|
|
64de77c065 | ||
|
|
19ba1e723c | ||
|
|
1d4dc44668 | ||
|
|
0fe3435033 | ||
|
|
6d9c856514 | ||
|
|
9634838f64 | ||
|
|
28273ffc2a | ||
|
|
7b65b64864 | ||
|
|
b90fd59544 | ||
|
|
d5ee2d7ef1 | ||
|
|
73358e6bae | ||
|
|
f2ad41e8e2 | ||
|
|
187dc188c2 | ||
|
|
ea2d3cf597 | ||
|
|
bd30c4e05a | ||
|
|
51283a626c | ||
|
|
01601d210b | ||
|
|
99b58c0c4f | ||
|
|
1001c25487 | ||
|
|
c0ace5a0ca | ||
|
|
1041c4bba8 | ||
|
|
3db594af86 | ||
|
|
71ba0a239f | ||
|
|
13de1dab82 | ||
|
|
8157673291 | ||
|
|
2697b46616 | ||
|
|
81aeb0619b | ||
|
|
693ac693a4 | ||
|
|
b22d034933 | ||
|
|
bb7c112a2d | ||
|
|
0101aef51a | ||
|
|
9eef5b50e6 | ||
|
|
2bda4cdaf8 | ||
|
|
b708a90548 | ||
|
|
90774f1442 | ||
|
|
a705aa36b7 | ||
|
|
60971f1af4 | ||
|
|
00dee37e22 | ||
|
|
2c16cc7116 | ||
|
|
5ad5c0df7e | ||
|
|
00328cbd0a | ||
|
|
7f22bb0612 | ||
|
|
7ebe8c0891 | ||
|
|
dbc0c8c6d2 | ||
|
|
04a0dcb39a | ||
|
|
24710af156 | ||
|
|
e149046272 | ||
|
|
1381a526ab | ||
|
|
fa7af04418 | ||
|
|
95050850d6 | ||
|
|
04aba81ab3 | ||
|
|
a11e3d73a7 | ||
|
|
793a75b8a2 | ||
|
|
976fdf6e50 | ||
|
|
a71723e51f | ||
|
|
7a3e8dd378 | ||
|
|
2cd1384786 | ||
|
|
9f32644c32 | ||
|
|
a7387c27fb | ||
|
|
2f8b7276a4 | ||
|
|
cbcfbfaaad | ||
|
|
a2debe972f | ||
|
|
57dad6d706 | ||
|
|
1f4b751fbb | ||
|
|
8808cbfc04 | ||
|
|
c6b56d580e | ||
|
|
b358dfcf58 | ||
|
|
57e2b0d670 | ||
|
|
b7df54e4f3 | ||
|
|
2ccc93bb14 | ||
|
|
2c1fd4c8bf | ||
|
|
466081aa66 | ||
|
|
be6bb3ffc5 | ||
|
|
17216a354d | ||
|
|
d1deac1bbb | ||
|
|
44275a8a04 | ||
|
|
e64d26af6c | ||
|
|
8ca52c64b9 | ||
|
|
ec3084db12 | ||
|
|
faa4a06c42 | ||
|
|
72d0e876c1 | ||
|
|
a04368e9fc | ||
|
|
a18f0cfc84 | ||
|
|
e1110b20df | ||
|
|
5b2ab93e60 | ||
|
|
533328aa91 | ||
|
|
6eb84c195a | ||
|
|
f8c4c93f20 | ||
|
|
e2b304daad | ||
|
|
c33f3c067b | ||
|
|
20c17648e6 | ||
|
|
240876e710 | ||
|
|
999dbcfdf3 | ||
|
|
67510b8fad | ||
|
|
24b526db93 | ||
|
|
dfe4c0d495 | ||
|
|
2b933c45dd | ||
|
|
f90b23ddd9 | ||
|
|
0eccd7b36e | ||
|
|
a95def9139 | ||
|
|
76ce424f8f | ||
|
|
6855dcb8fd | ||
|
|
e3ec54f213 | ||
|
|
aa3d92360e | ||
|
|
66f427dd2d | ||
|
|
25361fafdb | ||
|
|
081366a682 | ||
|
|
2100286dbb | ||
|
|
29e0daf80d | ||
|
|
57f3bf2807 | ||
|
|
775bdce6e1 | ||
|
|
b9b2a59915 | ||
|
|
33affc5a59 | ||
|
|
9adf760c8e | ||
|
|
eb24e60b8c | ||
|
|
9c744b34f8 | ||
|
|
6054f68754 | ||
|
|
74b579e281 | ||
|
|
c0da7d935d | ||
|
|
07f679b2e7 | ||
|
|
0c16c6db63 | ||
|
|
613634473a | ||
|
|
47861b0dc3 | ||
|
|
426ca2e5c7 | ||
|
|
91357a2d03 | ||
|
|
e4423200cc | ||
|
|
7d865b0a09 | ||
|
|
5b48561f36 | ||
|
|
a5be719261 | ||
|
|
77fb1abb85 | ||
|
|
b20a46ba81 | ||
|
|
9c150532fc | ||
|
|
4ab3387db4 | ||
|
|
92157aa5da | ||
|
|
801288c88c | ||
|
|
6f5c81c9a2 | ||
|
|
e5edffae4b | ||
|
|
3fac5751e4 | ||
|
|
f2d4d2fae1 | ||
|
|
e91a74d31f | ||
|
|
955f50b94e | ||
|
|
a8654280a7 | ||
|
|
083b5ec5ad | ||
|
|
500f9d92c8 | ||
|
|
67eabdc17c | ||
|
|
d9f917ecef | ||
|
|
eb4512aee3 | ||
|
|
0785a46fe3 | ||
|
|
78c9875d92 | ||
|
|
c135d46cbc | ||
|
|
4b6c5fe1cc | ||
|
|
66f9d6f435 | ||
|
|
4980b62dc2 | ||
|
|
d91fa89310 | ||
|
|
d6debc5408 | ||
|
|
c3ed7262cd | ||
|
|
455fb55857 | ||
|
|
1370a29233 | ||
|
|
bb5daad3cc | ||
|
|
6be765a9bf | ||
|
|
8eeb4c3d6c | ||
|
|
d0ff0c7d5c | ||
|
|
c431370a68 | ||
|
|
4f598f7422 | ||
|
|
7b1b017d8c | ||
|
|
1afeaf0e15 | ||
|
|
33e696071e | ||
|
|
44cf4e7516 | ||
|
|
2aa9302751 | ||
|
|
6cdb0d1eab | ||
|
|
d9225d06d9 | ||
|
|
37784bce6c | ||
|
|
073352b21e | ||
|
|
f4fd17be4d | ||
|
|
a011cd5e3b | ||
|
|
3a45c18555 | ||
|
|
38cd3edf99 | ||
|
|
e406bd445d | ||
|
|
f1603349cc | ||
|
|
d5c36db531 | ||
|
|
0f92609425 | ||
|
|
0f4d381ac0 | ||
|
|
ea0c713c89 | ||
|
|
4d00e37ca1 | ||
|
|
1e324636e0 | ||
|
|
df3c4bb369 | ||
|
|
2fefb56b8c | ||
|
|
934c0b56be | ||
|
|
e0cf83c944 | ||
|
|
9cabb04395 | ||
|
|
2ab7d3d98a | ||
|
|
228aae31c9 | ||
|
|
86a0ba55fe | ||
|
|
9e3755f9ca | ||
|
|
ab794db0ab | ||
|
|
b445ddc2c8 | ||
|
|
fb50f94ed8 | ||
|
|
b02857cf04 | ||
|
|
be306cf86c | ||
|
|
bf7a2cc911 | ||
|
|
4989486f45 | ||
|
|
9be6055113 | ||
|
|
8b0c2f6d1b | ||
|
|
141e2ca37c | ||
|
|
b803fefebd | ||
|
|
ba1979a662 | ||
|
|
1b46033584 | ||
|
|
7b2f79e3ac | ||
|
|
9145a66cef | ||
|
|
d5944c2443 | ||
|
|
d135dff5cd | ||
|
|
62a838ebdd | ||
|
|
f2d0e18f2c | ||
|
|
4347fc5f07 | ||
|
|
ce0cbd868c | ||
|
|
2658082607 | ||
|
|
55aa7f968f | ||
|
|
0882942164 | ||
|
|
917ff43795 | ||
|
|
d562644581 | ||
|
|
02aa73bb8d | ||
|
|
0fa871ef61 | ||
|
|
815ceef276 | ||
|
|
10c1705790 | ||
|
|
e467cdaf23 | ||
|
|
269ef9b2de | ||
|
|
657ee435d1 | ||
|
|
2f8af3a1b7 | ||
|
|
2c5e412d91 | ||
|
|
9d4a62fa5e | ||
|
|
714d97dd49 | ||
|
|
5a813eb4c8 | ||
|
|
185abdb442 | ||
|
|
60994c7667 | ||
|
|
eacea56d9d | ||
|
|
99486b1eaf | ||
|
|
aeb8c01370 | ||
|
|
2ad3a1bfed | ||
|
|
329bf25b14 | ||
|
|
90eee5674f | ||
|
|
1761087283 | ||
|
|
a13c5541cc | ||
|
|
262cd92300 | ||
|
|
87fa2693f0 | ||
|
|
64a28acd64 | ||
|
|
3ea80ef944 | ||
|
|
e1bbecad38 | ||
|
|
149d98a10b | ||
|
|
bd583b58dc | ||
|
|
7d0f7332b9 | ||
|
|
83508cfb1b | ||
|
|
969a85f303 | ||
|
|
961c57c6f0 | ||
|
|
89d6219e95 | ||
|
|
131c9ffc13 | ||
|
|
b1fdaced9f | ||
|
|
0cf208b80a | ||
|
|
9c8eb8a2bc | ||
|
|
452bf0e544 | ||
|
|
9d189e9dbf | ||
|
|
a3098ee67c | ||
|
|
73305a0cd1 | ||
|
|
e58f0352a7 | ||
|
|
8d393a1e1c | ||
|
|
d0b3588f6c | ||
|
|
7625e923e5 | ||
|
|
76ad744581 | ||
|
|
47110730e6 | ||
|
|
7f01b3040d | ||
|
|
f30f3263bd | ||
|
|
ca678b1784 | ||
|
|
ae7555a423 | ||
|
|
e0925b9d38 | ||
|
|
563ea0eb78 | ||
|
|
198532e280 | ||
|
|
69ffb5e1dc | ||
|
|
237156c46f | ||
|
|
264ba57cbb | ||
|
|
a537ae4217 | ||
|
|
8896e05e35 | ||
|
|
c617f64b6b | ||
|
|
eb36641523 | ||
|
|
885cceccae | ||
|
|
2d07c7d80e | ||
|
|
e02dc1be15 | ||
|
|
c34224effa | ||
|
|
987313e7dc | ||
|
|
6a959fb5e4 | ||
|
|
b00688bad1 |
+21
-2
@@ -3,14 +3,21 @@
|
||||
# Templates (*.template) are safe and remain tracked.
|
||||
Configurations/host*.conf
|
||||
Configurations/master.conf
|
||||
Configurations/*.bak
|
||||
# *.bak alone missed conf_upgrade's real output — it writes host1.conf.bak-20260802, which does
|
||||
# not end in .bak — so those sat untracked rather than ignored, one `git add -A` from being
|
||||
# pushed. The glob has to cover the suffix.
|
||||
Configurations/*.bak*
|
||||
.vscode
|
||||
|
||||
# ── Personal scratch notes — dev-only, never pushed ───────────────────────────
|
||||
Notes_To-Do.md
|
||||
|
||||
# ── Runtime state, data, logs ─────────────────────────────────────────────────
|
||||
data/
|
||||
# Contents, not the directory itself. Ignoring "data/" outright means git never descends into
|
||||
# it, and a negation for a file inside an excluded directory is silently ineffective — so the
|
||||
# README explaining what data/ is would be the one file missing from every installation of it.
|
||||
data/*
|
||||
!data/README.md
|
||||
State_Files/
|
||||
.cache/
|
||||
*.log
|
||||
@@ -36,3 +43,15 @@ Plugin/dist/
|
||||
*.swp
|
||||
*~
|
||||
.vscode/
|
||||
|
||||
# ── Local-only plugin surfaces (per-installation, never pushed) ───────────────
|
||||
# Varaverk.page discovers pages/local/*.php and registers each as a tab; api/local/ holds their
|
||||
# endpoints. Both are symlinks into a store outside this repo, so what they contain belongs to
|
||||
# one installation and is not part of the project. The tracked loader is deliberately generic —
|
||||
# it names no page — so the public mirror never learns what a given server runs here.
|
||||
#
|
||||
# No trailing slash on either pattern. These paths are symlinks, not directories, and git treats
|
||||
# a symlink as a blob — a "dir/" pattern does not match one, so the entries sat untracked rather
|
||||
# than ignored, which is the same near-miss the *.bak rule above documents.
|
||||
Plugin/unraid/pages/local
|
||||
Plugin/unraid/api/local
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# The index is generated, host-specific, and regenerable in minutes. Never commit it.
|
||||
*.db
|
||||
*.db-wal
|
||||
*.db-shm
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
# ━━━━━ AI ━━━━━
|
||||
|
||||
Retrieval over Varaverk's own documentation. Ask the system a question about itself and get an
|
||||
answer grounded in its actual headers, READMEs, Manuals and conf templates — with sources.
|
||||
|
||||
Everything here is **off by default and optional**. Varaverk works exactly as well with
|
||||
`AI_ENABLED=false` as with it true. Nothing in the ecosystem depends on this folder.
|
||||
|
||||
| File | Role |
|
||||
|------|------|
|
||||
| `ai_index.sh` | Build / refresh the retrieval index |
|
||||
| `ai_query.sh` | Ask a question, or search the index directly |
|
||||
| `lib/chunk.js` | Split repo files into retrieval units |
|
||||
| `lib/index.js` | Embed chunks, store vectors in SQLite |
|
||||
| `lib/search.js` | Embed a query, score it, rank results |
|
||||
| `lib/cli.js` | Argument bridge the bash entry points call |
|
||||
|
||||
---
|
||||
|
||||
## ━━━ WHY THIS WORKS AT ALL ━━━
|
||||
|
||||
**Because the header audit came first.**
|
||||
|
||||
The single worst failure in naive retrieval is a chunk that contains half of one idea and half
|
||||
of another — a fixed-size window cutting mid-thought, embedding two unrelated things as one
|
||||
vector. That problem does not exist here, because every script in the repo carries the same six
|
||||
sections at exact, greppable boundaries:
|
||||
|
||||
```
|
||||
PURPOSE → OPERATIONAL MODEL → DESIGN PRINCIPLES → OPERATIONAL SAFEGUARDS
|
||||
→ CONFIGURATION → RUNTIME MODES
|
||||
```
|
||||
|
||||
Split on those and every chunk is a coherent unit by construction. No token windows, no overlap
|
||||
heuristics, no tuning.
|
||||
|
||||
**And because the headers say *why*.** A model can read `mover_stop.sh` and describe what it
|
||||
does. It cannot look at that code and know the cache writers are lockless *on purpose*, or that
|
||||
`removeCompletedDownloads` being true on both arrs is intentional. Those live in
|
||||
`DESIGN PRINCIPLES` and `OPERATIONAL SAFEGUARDS`, which is precisely what makes this index worth
|
||||
more than an equivalent pile of source.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ SECTION ROUTING ━━━
|
||||
|
||||
Every chunk stores its section name as its own column, so a question's *shape* can steer
|
||||
retrieval before similarity is even considered:
|
||||
|
||||
| Question shape | Steered toward |
|
||||
|----------------|----------------|
|
||||
| "what stops X and Y overlapping" | `OPERATIONAL SAFEGUARDS` |
|
||||
| "which variable controls X" | `CONFIGURATION` |
|
||||
| "does this take --dry-run" | `RUNTIME MODES` |
|
||||
| "why is it built this way" | `DESIGN PRINCIPLES` |
|
||||
| "how does X work" | `OPERATIONAL MODEL` |
|
||||
|
||||
Applied as a **score boost, not a filter**. Intent detection is a heuristic, and a heuristic
|
||||
must never be able to exclude the one chunk that holds the answer. `--section=NAME` forces a
|
||||
hard filter when you actually want one.
|
||||
|
||||
The boost still has a blind spot worth knowing about: **definitional questions.** "What is
|
||||
Varaverk?" matches the `PURPOSE` intent, so every script's one-line PURPOSE gets boosted above
|
||||
the top-level prose that actually answers it — and the model correctly replies that the context
|
||||
does not define the system. The corpus is fine; `README.md` is indexed. The routing simply
|
||||
buries it. `--kind=readme` is the hard filter for that case:
|
||||
|
||||
```bash
|
||||
bash AI/ai_query.sh --kind=readme "what is Varaverk"
|
||||
```
|
||||
|
||||
`--kind` filters on where a chunk came from — `header`, `readme`, `manual`, `template`, `doc`, `ui` —
|
||||
and composes with `--section`. Prefer it over `--section` for "what is" and "why does this
|
||||
exist" questions, where the answer is narrative rather than a header field.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ NAMED-PARAGRAPH SUB-CHUNKING ━━━
|
||||
|
||||
Sections alone were not granular enough, and the failure was instructive.
|
||||
|
||||
`rsync.sh` documents fourteen distinct safeguards in one 2.8k-character
|
||||
`OPERATIONAL SAFEGUARDS` block. Asked *"what happens if pass 1 of a merge run fails"*, the
|
||||
correct chunk scored **0.558** — below unrelated chunks from other files — because the other
|
||||
thirteen safeguards dominated the vector.
|
||||
|
||||
The header convention writes each safeguard as a named paragraph: an unindented title, an
|
||||
indented body. Splitting on those titles took the same query to **0.718** and first place.
|
||||
|
||||
```
|
||||
Merge-Run Delete Interlock ← its own chunk
|
||||
--delete is applied only when pass 1 completed...
|
||||
```
|
||||
|
||||
The parent section name is carried onto every sub-chunk, so routing still works. The title
|
||||
detection requires the *next* line to be indented — without that check, any wrapped prose line
|
||||
became a spurious boundary mid-sentence.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ WHAT IS INDEXED ━━━
|
||||
|
||||
Roughly 2,900 chunks across ~180 files:
|
||||
|
||||
| Kind | Source |
|
||||
|------|--------|
|
||||
| `header` | Script headers — bash and PHP, sections and named paragraphs |
|
||||
| `readme` | Every `README-*.md` |
|
||||
| `manual` | Every `Manual-*.md` |
|
||||
| `template` | `Deployment/*.template` — the versioned conf schema |
|
||||
| `doc` | Top-level `README.md`, `Manual.md`, design notes |
|
||||
| `ui` | `Plugin/unraid/pages/readme/*.md` — the WebGUI's own help panels |
|
||||
|
||||
**Script bodies are not indexed.** Headers state intent, code states mechanism; for the
|
||||
questions this answers, intent retrieves better and costs far less.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ THE SAFETY BOUNDARY ━━━
|
||||
|
||||
**Only `git ls-files` is ever indexed.** This is not a convenience — it is the security model.
|
||||
|
||||
`Configurations/` and `data/` are gitignored, so every file holding a credential
|
||||
was never in the repo to begin with. The index therefore describes the full conf schema (via the
|
||||
tracked templates, which carry all the explanatory comments) while structurally **never
|
||||
containing a secret**.
|
||||
|
||||
> Do not "improve" this into a filesystem walk. A logged secret can be rotated. A secret
|
||||
> averaged into a 768-dimension float cannot be found, let alone removed.
|
||||
|
||||
A live conf value the model genuinely needs should arrive through a tool call at query time,
|
||||
subject to redaction — never baked into a vector at index time.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ HOW IT IS BUILT ━━━
|
||||
|
||||
**SQLite, no vector database.** ~2,900 chunks × 768 dims is a couple of million multiply-adds
|
||||
per query — under a millisecond. A vector DB would be a container to run, monitor, fail over and
|
||||
back up, in exchange for nothing at this scale.
|
||||
|
||||
**Vectors are raw little-endian float32 BLOBs.** `nomic-embed-text` returns L2-normalised
|
||||
vectors, so cosine similarity is a plain dot product — no normalising, no magnitude cache. PHP
|
||||
reads the same blobs with `unpack('f*', $blob)` when the UI needs them.
|
||||
|
||||
**Incremental on mtime.** A file whose mtime has not moved is skipped without being read. A
|
||||
no-op refresh takes about 70 ms; a full rebuild takes a few minutes.
|
||||
|
||||
**Node for the maths, bash for everything else.** The bash entry points own configuration,
|
||||
gating, locking and logging exactly as every other Varaverk job does. Node owns only float
|
||||
vector maths and SQLite BLOBs. Same split as `api_cache_writer.sh` and its PHP.
|
||||
|
||||
> `lib/index.js` uses `node:sqlite`, which Node still marks experimental. It is used because it
|
||||
> needs no native compilation on Unraid. If a Node upgrade ever breaks it, the index is
|
||||
> regenerable in minutes — this is a disposable artefact, not a datastore.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ USAGE ━━━
|
||||
|
||||
```bash
|
||||
# Build (needs AI_ENABLED=true)
|
||||
bash AI/ai_index.sh # incremental
|
||||
bash AI/ai_index.sh --force # full rebuild
|
||||
bash AI/ai_index.sh --dry-run # what would be indexed; contacts nothing
|
||||
bash AI/ai_index.sh --status # size, counts, last build
|
||||
|
||||
# Ask
|
||||
bash AI/ai_query.sh "what stops rsync and the mover running at once"
|
||||
bash AI/ai_query.sh --search "why are the cache writers lockless"
|
||||
bash AI/ai_query.sh --section=CONFIGURATION "which variable sets the mover grace period"
|
||||
bash AI/ai_query.sh --kind=readme "what is Varaverk" # definitional / narrative
|
||||
bash AI/ai_query.sh --json "..." # for other scripts
|
||||
```
|
||||
|
||||
**`--search` is the trustworthy mode.** It returns verbatim repo text with nothing generated.
|
||||
When the answer matters, use it — or read the sources the generated answer cites.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ TRUST THE SOURCES, NOT THE PROSE ━━━
|
||||
|
||||
The generation prompt instructs the model to answer only from retrieved context and to say what
|
||||
is missing rather than fill the gap from general knowledge. That instruction matters here more
|
||||
than usual: this repo's conventions are frequently *not* the conventional ones, and a confident
|
||||
generic answer about rsync, Docker or systemd is worse than no answer.
|
||||
|
||||
It works — and its faithfulness cut both ways on the first real test. Asked which variable
|
||||
controls the mover's grace window, the model answered `MOVER_STOP_TIMEOUT`, "defaults to 30
|
||||
seconds", citing `mover_stop.sh › CONFIGURATION`. The variable was right. The 30 was wrong — the
|
||||
real value is 300 — and the model was quoting the header verbatim. **The header was stale.**
|
||||
|
||||
That sweep then found six stale `(default: N)` claims across the repo, all since corrected. The
|
||||
lesson is the operating principle for this whole folder:
|
||||
|
||||
> Retrieval is exactly as accurate as the documentation it points at. When an answer looks
|
||||
> wrong, check the cited source before blaming the model — it is usually reporting a real
|
||||
> problem in the repo.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ WHAT THIS DOES NOT DO ━━━
|
||||
|
||||
- **It does not write conf.** `AI_CONF_WRITE_ENABLED` exists in `master.conf` and is off, with
|
||||
an empty key whitelist. Nothing in this folder writes a setting.
|
||||
- **It does not make decisions.** No watchdog, cleanup or fallback path consults it. The
|
||||
per-feature `AI_ASSIST_*` toggles are all false and each one is earned separately.
|
||||
- **It does not index code.** Headers and docs only.
|
||||
- **It does not sync.** The index is host-local and gitignored. Each node builds its own.
|
||||
- **It is not required.** Every script runs identically with `AI_ENABLED=false`.
|
||||
|
||||
See `Notes_AI-Design.md` at the repo root for the wider design — host resolution across the
|
||||
Tailscale mesh, per-feature rollout tiers, and the conf-write guardrails that would have to be
|
||||
built before any of that is enabled.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ SCHEDULING ━━━
|
||||
|
||||
The index tracks the repo automatically, from the only moment the corpus actually changes — a
|
||||
successful pull. `git_pull_execute.sh` re-indexes behind three gates: the pull succeeded and
|
||||
changed tracked files, `AI_INDEX_ON_PULL=true`, and `AI_ENABLED=true`. It is never fatal — a
|
||||
failed index leaves the previous one in place and the pull still reports success.
|
||||
|
||||
No cron entry and no `DAILY_MAINTENANCE_SCRIPTS` line are needed; the daily pull carries it.
|
||||
|
||||
An incremental run on an unchanged repo is ~70 ms, so a daily entry costs effectively nothing
|
||||
and a pull that changed twelve files costs a few seconds.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ PROFILES ━━━
|
||||
|
||||
A profile is a contract plus a set of inputs. `Plugin/unraid/include/ai_profiles.php` is the one
|
||||
definition of both, read by the endpoint, the worker, the shared chat include and the Scheduler
|
||||
dock.
|
||||
|
||||
| Profile | Turns | Retrieves | Notes |
|
||||
|---|---|---|---|
|
||||
| `varaverk` | 3 | yes | answers only from the index, with citations. The default. |
|
||||
| `chat` | 8 | **no** | ordinary conversation. Holds zero capabilities, deliberately. |
|
||||
| `code` | 4 | no | drafts shell for Custom Scripts; scans its own output for destructive ops |
|
||||
| `troubleshoot` | 2 | yes | reasons from evidence — an open log, or one you name. May file bug reports. |
|
||||
|
||||
Capabilities are granted per profile — retrieval, live health, run evidence, scoped log,
|
||||
incidents, conf lookup, bug filing, code scanning. `chat` holding an empty list is a guarantee,
|
||||
not an oversight: anything added to it stops being general chat and becomes an assistant that
|
||||
sometimes invents claims about this installation.
|
||||
|
||||
### Routing out of General Chat
|
||||
|
||||
`chat` hands a question to whichever profile fits, decided by `vv_ai_route_from_chat()`. Ordered
|
||||
most specific first, because these overlap on purpose:
|
||||
|
||||
| Question | Goes to | Why |
|
||||
|---|---|---|
|
||||
| "write me a script that prunes logs" | `code` | asked for something written |
|
||||
| "why did the daily orch fail" | `troubleshoot` | diagnostic phrasing **and** something here to diagnose |
|
||||
| "how did the daily orch go" | `varaverk` | about this install, but not a fault |
|
||||
| "what does arr_sync.sh do" | `varaverk` | names a script, wants documentation |
|
||||
| "why is the sky blue" | stays `chat` | diagnostic phrasing about nothing here |
|
||||
|
||||
`code` is checked first because it is the only intent about a thing that does not exist yet, so
|
||||
nothing else can claim it — and it is anchored on the verb, which is what keeps "write me a
|
||||
script" apart from "what does this script do". Escalation adds capability, so a wrong escalation
|
||||
costs more than a missed one: anything unrecognised stays in `chat`, the profile that cannot
|
||||
invent claims about this system. The worker reverts to `chat` anyway if retrieval comes back
|
||||
empty.
|
||||
|
||||
The answer opens with one line naming the profile that took it, because the button still shows
|
||||
the one you picked and an answer arriving under a different contract otherwise reads as the
|
||||
assistant ignoring you.
|
||||
|
||||
Routing is asserted by `Plugin/unraid/Tools/ai_explain_check.sh` against
|
||||
`ai_explain_fixtures.txt` — every case runs through the worker's `--explain` mode, which stops
|
||||
where deterministic assembly ends and never calls the model.
|
||||
|
||||
This used to live in five places — history depth in the endpoint, capabilities in `include/ai.php`,
|
||||
label and depth again in JavaScript, a prompt branch in the worker, and a label map on the
|
||||
Scheduler page. They had already drifted: the JavaScript knew three profiles where PHP knew four.
|
||||
The system prompts still live in `Tools/ai_chat_worker.php`, because they have one reader and
|
||||
moving them would relocate the most delicate text in the subsystem without removing a duplicate.
|
||||
|
||||
## ━━━ CONVERSATIONS ━━━
|
||||
|
||||
Chats are stored server-side under `AI_DATA_DIR/ai_chats/`, one JSON file each, saved
|
||||
automatically when a turn completes and pruned to `AI_CHAT_HISTORY_MAX` (default 10, oldest
|
||||
first by creation).
|
||||
|
||||
There is no Save button. A conversation worth keeping is not reliably one you knew was worth
|
||||
keeping while you were having it.
|
||||
|
||||
The same store backs the AI tab and the Monitor tab's AI row, so a thread started on the
|
||||
dashboard is the one you carry on in the tab. Messages are re-validated per message on the way
|
||||
in — a stored chat is replayed into a later prompt when reopened, so an unchecked role written
|
||||
there would be an injection that survives a reload rather than one turn.
|
||||
|
||||
Reopened chats render as plain turns: sources, reasoning and timings describe one generation and
|
||||
are not stored, because redrawing them beside a transcript that may be continued under a
|
||||
different profile would be citing evidence for an answer no longer being made.
|
||||
|
||||
### Secrets are redacted on the way to disk
|
||||
|
||||
A conversation about settings is a conversation containing credentials — asking for an API key to
|
||||
be changed means typing one. Message bodies are redacted in `vv_ai_chat_save()`, and the question
|
||||
is redacted again before it reaches `ai.log`.
|
||||
|
||||
**On the way out, never in flight.** The live turn keeps the real value, because the model needs
|
||||
it to carry out what was asked. What it does not need is that value still in the transcript a week
|
||||
later — and a stored chat is replayed into a later prompt when reopened, so an unredacted one
|
||||
would hand the credential back on every subsequent turn, indefinitely.
|
||||
|
||||
Two passes, because they catch different things:
|
||||
|
||||
| Pass | Catches | Method |
|
||||
|---|---|---|
|
||||
| Known values | a credential this host already holds | exact match against secret-shaped conf keys, longest first |
|
||||
| Assignment shapes | a credential arriving that is not in the conf yet | `NAME=value`, `"api_key": value`, "set the token to …" |
|
||||
|
||||
The second pass is the one that matters for settings changes: *"change the Emby API key to X"* is
|
||||
a secret arriving, and X matches nothing on disk until after the write it is requesting.
|
||||
|
||||
Ordinary prose is left alone — the patterns anchor on a secret-shaped *name*, so `CACHE_WARN_GB=100`
|
||||
and "turn off the zfs scrub" pass through untouched. `vv_conf_key_is_secret()` is shared with the
|
||||
conf audit log, so the two cannot disagree about what counts as a secret.
|
||||
|
||||
## ━━━ TOKEN ACCOUNTING ━━━
|
||||
|
||||
Every completed `ask` appends one row to `AI_TOKEN_DB` (`data/ai/ai_token_history.db`):
|
||||
|
||||
```
|
||||
date|time|host|profile|source|prompt_tokens|completion_tokens|tok_s
|
||||
2026-08-04|22:03:51|host1|varaverk|cli|2041|318|61.4
|
||||
```
|
||||
|
||||
Both paths write it — this CLI (`source=cli`) and the WebGUI worker (`source=webgui`) — so the
|
||||
totals are not quietly the tab's alone. `ai_query.sh` passes `--token-db` and `--token-host`;
|
||||
called by hand without them, `cli.js` simply skips the row rather than guessing a path, because
|
||||
this file never reads conf itself.
|
||||
|
||||
Read it on the plugin's AI tab, which aggregates today / last 7 days / all time, per host. Or
|
||||
straight from the shell, since it is just a delimited file:
|
||||
|
||||
```bash
|
||||
# tokens used today
|
||||
awk -F'|' -v d="$(date +%F)" '$1==d {p+=$6; c+=$7} END {print p+c}' data/ai/ai_token_history.db
|
||||
```
|
||||
|
||||
**The host column is where the turn ran, not where the file is read.** Each host writes only its
|
||||
own rows.
|
||||
|
||||
`ai_token_sync.sh` pulls each partner's ledger into `$AI_TOKEN_CACHE_DIR/<slot>.tokens.db` (`/tmp/varaverk/ai/`) — the
|
||||
same trick `conf_sync.sh` uses for partner confs, and it runs from
|
||||
`INTERMEDIATE_MAINTENANCE_SCRIPTS` every four hours. The tab then reads every ledger it can see,
|
||||
so a fleet total is a fleet total.
|
||||
|
||||
Pull only, no push: nothing here is needed by anyone else, and a reader that fetches its own data
|
||||
controls its own freshness instead of depending on the partner's cron. A partner file may only
|
||||
contribute rows whose `host` column matches its filename — a ledger copied into the wrong slot
|
||||
would otherwise be double-counted against a total that still looked plausible.
|
||||
|
||||
The cache is tmpfs with **no save/restore pair**, unlike the conf cache. Stale counters are worse
|
||||
than absent ones: absent renders as "not collected here", stale renders as fact. An unreachable
|
||||
partner leaves its file alone and logs at info, because a partner being down for weeks is a
|
||||
normal state, not an incident.
|
||||
|
||||
Pruning is by row count (`AI_TOKEN_RETAIN_ROWS`, default 20000) and happens on write, but only
|
||||
once the file passes a size threshold — an ordinary turn costs a `stat()` and an append. The CLI
|
||||
deliberately does not prune: duplicating a read-modify-write of the whole file in a second
|
||||
language is how the two drift apart.
|
||||
Executable
+265
@@ -0,0 +1,265 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================== AI Retrieval Index Builder ========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ==============================================================================================
|
||||
# Builds and refreshes the retrieval index over Varaverk's own documentation — every script
|
||||
# header, folder README and Manual, and the conf templates. Chunks each file on the section
|
||||
# boundaries the header convention already defines, embeds each chunk through Ollama, and
|
||||
# stores the vectors in SQLite for AI/ai_query.sh to search.
|
||||
#
|
||||
# The index is derived data. It is gitignored, host-local, and rebuildable from the repo in
|
||||
# minutes — nothing depends on it surviving.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
# Chunking mirrors the header convention rather than using a fixed token window:
|
||||
#
|
||||
# bash headers split on the six section names, then sub-split named-paragraph
|
||||
# safeguards and principles so one question finds one answer
|
||||
# PHP headers same six names plus the per-layer tails (EXPORTS, RENDERS, ...)
|
||||
# markdown split on headings
|
||||
# conf templates split on the ━━━ section rules
|
||||
#
|
||||
# Every chunk keeps its section name as a field, which is what lets a query about a safeguard
|
||||
# be steered toward OPERATIONAL SAFEGUARDS chunks before similarity is considered.
|
||||
#
|
||||
# Incremental by file mtime. A file whose mtime has not moved is skipped without being read,
|
||||
# so a routine refresh costs well under a second and a full rebuild costs a few minutes.
|
||||
#
|
||||
# Heavy lifting runs in Node — float vector maths and SQLite BLOBs are genuinely awkward in
|
||||
# bash. This follows the api_cache_writer.sh precedent: a bash shim owning config, gating,
|
||||
# locking and logging, in front of the language that fits the work.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Tracked Files Only
|
||||
# Indexes exactly what `git ls-files` reports. Configurations/, State_Files/ and data/ are
|
||||
# gitignored, so it is structurally impossible for a credential to reach the index — the
|
||||
# files holding them were never in the repo. This must never become a filesystem walk: a
|
||||
# secret written into a vector cannot be rotated back out of it.
|
||||
#
|
||||
# The Index Is Disposable
|
||||
# Stored under DATA_DIR, gitignored, and never synced to a partner. Losing it costs one
|
||||
# rebuild. Nothing reads it as a source of truth — it points at files, and the files are
|
||||
# the truth.
|
||||
#
|
||||
# Documentation Is The Corpus, Not The Code
|
||||
# Script bodies are not indexed. The headers state intent and the code states mechanism;
|
||||
# for the questions this answers, intent retrieves far better and is far cheaper.
|
||||
#
|
||||
# Off By Default
|
||||
# Does nothing unless AI_ENABLED is true. A node with AI off never pays for this.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Fail-Closed Gate
|
||||
# Exits cleanly unless AI_ENABLED is exactly "true". Any other value, including unset,
|
||||
# means off.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Writes into DATA_DIR alongside other Varaverk state.
|
||||
#
|
||||
# Concurrency Lock
|
||||
# acquire_lock() prevents two indexers racing on the same database.
|
||||
#
|
||||
# Dependency Verification
|
||||
# Verifies node and the Ollama endpoint before touching the database. A missing dependency
|
||||
# is reported and exits non-zero rather than leaving a half-built index.
|
||||
#
|
||||
# Reachability Pre-flight
|
||||
# Probes the resolved Ollama URL with AI_CONNECT_TIMEOUT before starting. An unreachable
|
||||
# endpoint aborts immediately instead of failing once per batch across the whole corpus.
|
||||
#
|
||||
# Partial Failure Is Not Recorded As Success
|
||||
# A file whose chunks all failed to embed keeps its previous rows and its old mtime, so the
|
||||
# next run retries it. A run with any failed batch exits 3.
|
||||
#
|
||||
# Atomic Per-Run Write
|
||||
# All database changes commit in one transaction. An interrupted run leaves the previous
|
||||
# index intact rather than a partially rewritten one.
|
||||
#
|
||||
# Deleted Files Are Removed From The Index
|
||||
# A file that has left the repo has its chunks deleted, so retrieval cannot cite something
|
||||
# that no longer exists.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# AI_ENABLED
|
||||
# Master switch. Fail-closed — must be exactly "true".
|
||||
#
|
||||
# AI_INDEX_DB
|
||||
# SQLite index path. (shipped default: $DATA_DIR/ai_index.db)
|
||||
#
|
||||
# AI_INDEX_BATCH
|
||||
# Chunks per embed request. (shipped default: 32)
|
||||
#
|
||||
# AI_CONNECT_TIMEOUT
|
||||
# Seconds for the reachability probe. (shipped default: 5)
|
||||
#
|
||||
# AI_REQUEST_TIMEOUT
|
||||
# Seconds for a single embed batch. (shipped default: 240)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_OLLAMA_URL
|
||||
# This host's Ollama endpoint. Empty means no local Ollama.
|
||||
#
|
||||
# HOST*_OLLAMA_EMBED_MODEL
|
||||
# Embedding model. The generation model cannot embed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ai_index.sh
|
||||
# Incremental refresh — only files whose mtime moved are re-embedded.
|
||||
#
|
||||
# ai_index.sh --force
|
||||
# Full rebuild. Discards the existing index and re-embeds every chunk.
|
||||
#
|
||||
# ai_index.sh --dry-run
|
||||
# Report what would be indexed. Contacts nothing and writes nothing.
|
||||
#
|
||||
# ai_index.sh --status
|
||||
# Show index location, size, chunk counts by kind and section, and last build time.
|
||||
#
|
||||
# ai_index.sh --log
|
||||
# Verbose — per-batch embedding progress.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../load_config.sh"
|
||||
|
||||
detect_hosts
|
||||
|
||||
DRY_RUN=false; FORCE=false; STATUS=false; LOG=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
--force) FORCE=true ;;
|
||||
--status) STATUS=true ;;
|
||||
--log) LOG=true ;;
|
||||
*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
CLI="${SCRIPT_DIR}/lib/cli.js"
|
||||
DB="${AI_INDEX_DB:-${DATA_DIR}/ai_index.db}"
|
||||
|
||||
_url_var="${MY_ID}_OLLAMA_URL"
|
||||
_emb_var="${MY_ID}_OLLAMA_EMBED_MODEL"
|
||||
OLLAMA_URL="${!_url_var:-}"
|
||||
EMBED_MODEL="${!_emb_var:-nomic-embed-text}"
|
||||
|
||||
# ── Status ────────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR AI Index Status"
|
||||
echo " Enabled: ${AI_ENABLED:-false}"
|
||||
echo " Database: $DB"
|
||||
if [[ -f "$DB" ]]; then
|
||||
echo " Size: $(du -h "$DB" 2>/dev/null | cut -f1)"
|
||||
echo " Chunks: $(sqlite3 "$DB" 'SELECT COUNT(*) FROM vv_chunks;' 2>/dev/null || echo '?')"
|
||||
echo " Files: $(sqlite3 "$DB" 'SELECT COUNT(*) FROM vv_files;' 2>/dev/null || echo '?')"
|
||||
_last=$(sqlite3 "$DB" "SELECT v FROM vv_meta WHERE k='last_index';" 2>/dev/null)
|
||||
[[ -n "$_last" ]] && echo " Last built: $(date -d "@$_last" '+%Y-%m-%d %H:%M:%S' 2>/dev/null)"
|
||||
echo " Model: $(sqlite3 "$DB" "SELECT v FROM vv_meta WHERE k='embed_model';" 2>/dev/null || echo '?')"
|
||||
echo " By kind:"
|
||||
sqlite3 "$DB" "SELECT ' '||kind||': '||COUNT(*) FROM vv_chunks GROUP BY kind ORDER BY COUNT(*) DESC;" 2>/dev/null
|
||||
else
|
||||
echo " Database: not built yet"
|
||||
fi
|
||||
echo " Ollama: ${OLLAMA_URL:-<none local>}"
|
||||
echo " Embed: $EMBED_MODEL"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Gate ──────────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "${AI_ENABLED:-false}" != "true" ]]; then
|
||||
log "AI_ENABLED is not true — skipping index build"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The mesh shares one AI, and the index belongs to the node that holds the model. A mirror has the
|
||||
# same checkout and could build one, but nothing there would read it: retrieval happens wherever
|
||||
# generation happens, which is the owner.
|
||||
#
|
||||
# A skip, not an error. This is reached from git_pull_execute.sh on every node after every pull;
|
||||
# before the AI became mesh-wide it ran here too and failed on the empty OLLAMA_URL, nightly and
|
||||
# silently, because the caller discards its output.
|
||||
_ai_owner="${AI_OWNER_HOST:-host1}"
|
||||
if [[ "${MY_ID,,}" != "${_ai_owner,,}" ]]; then
|
||||
log "This node is not the AI owner ($_ai_owner) — the index lives there; skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == false && "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
command -v node >/dev/null 2>&1 || { error "node not found — required to build the index"; exit 1; }
|
||||
[[ -f "$CLI" ]] || { error "missing $CLI"; exit 1; }
|
||||
|
||||
# The AI owner has had data/ai since the subsystem was built, so nothing ever created it — cli.js
|
||||
# opens the DB by path and does not make the directory. On a first build the failure surfaces as a
|
||||
# sqlite open error rather than as the missing directory it is.
|
||||
if [[ "$DRY_RUN" == false ]] && ! mkdir -p "$(dirname "$DB")"; then
|
||||
error "Cannot create $(dirname "$DB")"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$OLLAMA_URL" ]]; then
|
||||
error "${MY_ID}_OLLAMA_URL is empty — no local Ollama to index against"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Pre-flight: fail once, up front, rather than once per batch across the whole corpus.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if ! curl -sf --max-time "${AI_CONNECT_TIMEOUT:-5}" "${OLLAMA_URL%/}/api/tags" >/dev/null 2>&1; then
|
||||
error "Ollama unreachable at $OLLAMA_URL"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Build ─────────────────────────────────────────────────────────────────────────────────────
|
||||
_args=(
|
||||
index
|
||||
"--root=${SCRIPTS_DIR}"
|
||||
"--db=${DB}"
|
||||
"--url=${OLLAMA_URL}"
|
||||
"--model=${EMBED_MODEL}"
|
||||
"--batch=${AI_INDEX_BATCH:-32}"
|
||||
"--timeout=$(( ${AI_REQUEST_TIMEOUT:-240} * 1000 ))"
|
||||
)
|
||||
[[ "$FORCE" == true ]] && _args+=(--force)
|
||||
[[ "$DRY_RUN" == true ]] && _args+=(--dry-run)
|
||||
[[ "$LOG" == false ]] && _args+=(--quiet)
|
||||
|
||||
log "$ICON_GEAR Building AI index → $DB"
|
||||
node --no-warnings "$CLI" "${_args[@]}"
|
||||
_rc=$?
|
||||
|
||||
case "$_rc" in
|
||||
0) log "$ICON_DONE AI index build complete" ;;
|
||||
3) warn "AI index built with some batches failed — those files will retry next run" ;;
|
||||
*) error "AI index build failed (exit $_rc)" ;;
|
||||
esac
|
||||
|
||||
exit "$_rc"
|
||||
Executable
+209
@@ -0,0 +1,209 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================== AI Retrieval Query ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ==============================================================================================
|
||||
# Answers questions about Varaverk from Varaverk's own documentation. Embeds the question,
|
||||
# retrieves the closest chunks from the index AI/ai_index.sh built, and either prints them
|
||||
# directly or passes them to the generation model as grounding context.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
# Two modes over the same retrieval:
|
||||
#
|
||||
# --search print the matching chunks and their sources. No generation model involved,
|
||||
# so it is fast and its output is verbatim repo text.
|
||||
# (default) retrieve, then ask the generation model to answer strictly from what was
|
||||
# retrieved, citing each claim.
|
||||
#
|
||||
# Retrieval is steered by question shape. A question about what prevents something is pushed
|
||||
# toward OPERATIONAL SAFEGUARDS chunks, one about a variable toward CONFIGURATION, one asking
|
||||
# why toward DESIGN PRINCIPLES. This is a score boost, not a filter — a heuristic must not be
|
||||
# able to exclude the chunk that actually holds the answer.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Grounded Or Silent
|
||||
# The prompt instructs the model to answer only from retrieved context and to say what is
|
||||
# missing rather than fill the gap. This repo's conventions are frequently not the
|
||||
# conventional ones, and a confident generic answer about rsync or Docker is worse here
|
||||
# than no answer.
|
||||
#
|
||||
# Sources Are Always Shown
|
||||
# Every answer prints the chunks it drew on. An answer that cannot be traced back to a file
|
||||
# is not usable for changing anything.
|
||||
#
|
||||
# Search Is The Trustworthy Mode
|
||||
# --search returns repo text with nothing generated. When an answer matters, use it.
|
||||
#
|
||||
# Read-Only
|
||||
# Retrieves and answers. Nothing here writes conf, touches state, or runs another script.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Fail-Closed Gate
|
||||
# Exits cleanly unless AI_ENABLED is exactly "true".
|
||||
#
|
||||
# No Root Required
|
||||
# Reads the index and calls Ollama. Nothing it does needs privilege, so it does not ask for
|
||||
# any — this is the one AI script an ordinary user should be able to run.
|
||||
#
|
||||
# Missing Index Is Reported, Not Built
|
||||
# An absent index exits with guidance to run ai_index.sh. Building a corpus-wide index as a
|
||||
# side effect of a question would turn a two-second query into a several-minute one.
|
||||
#
|
||||
# Reachability Pre-flight
|
||||
# Probes Ollama with AI_CONNECT_TIMEOUT before embedding, so an unreachable endpoint fails
|
||||
# immediately with a clear message.
|
||||
#
|
||||
# Bounded Generation
|
||||
# The request is capped at AI_REQUEST_TIMEOUT. A wedged model cannot hang the caller.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# AI_ENABLED
|
||||
# Master switch. Fail-closed — must be exactly "true".
|
||||
#
|
||||
# AI_INDEX_DB
|
||||
# SQLite index to search. (shipped default: $DATA_DIR/ai_index.db)
|
||||
#
|
||||
# AI_SEARCH_K
|
||||
# Chunks retrieved per query. (shipped default: 8)
|
||||
#
|
||||
# AI_SEARCH_PER_FILE
|
||||
# Cap per file, so one document cannot fill the context. (shipped default: 3)
|
||||
#
|
||||
# AI_REQUEST_TIMEOUT
|
||||
# Seconds allowed for generation. (shipped default: 240)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_OLLAMA_URL / HOST*_OLLAMA_MODEL / HOST*_OLLAMA_EMBED_MODEL
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ai_query.sh "your question"
|
||||
# Retrieve and answer, with sources.
|
||||
#
|
||||
# ai_query.sh --search "your question"
|
||||
# Print matching chunks only. No generation.
|
||||
#
|
||||
# ai_query.sh --section=CONFIGURATION "your question"
|
||||
# Restrict retrieval to one header section.
|
||||
#
|
||||
# ai_query.sh --kind=readme "what is Varaverk"
|
||||
# Restrict retrieval to one chunk origin: header, readme, manual, template, doc.
|
||||
# Use this for definitional and narrative questions. Intent routing boosts header
|
||||
# sections such as PURPOSE, which answers "what does this script do" well but buries
|
||||
# the top-level prose that explains what the system *is*. --kind=readme reaches it.
|
||||
#
|
||||
# ai_query.sh --json "your question"
|
||||
# Machine-readable output for other scripts.
|
||||
#
|
||||
# ai_query.sh --status
|
||||
# Show index and endpoint state.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${SCRIPT_DIR}/../load_config.sh"
|
||||
|
||||
detect_hosts
|
||||
|
||||
SEARCH_ONLY=false; JSON=false; STATUS=false; SECTION=""; KIND=""; QUERY=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--search) SEARCH_ONLY=true ;;
|
||||
--json) JSON=true ;;
|
||||
--status) STATUS=true ;;
|
||||
--section=*) SECTION="${arg#*=}" ;;
|
||||
--kind=*) KIND="${arg#*=}" ;;
|
||||
--*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||
*) QUERY="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --kind is a hard filter on chunk origin; --section filters the header section within a
|
||||
# chunk. They answer different questions and compose: --kind=readme --section=PURPOSE is
|
||||
# meaningful. Validated here rather than in the CLI so a typo costs nothing — an unknown kind
|
||||
# silently matches no rows, which reads as "the index has no answer" and is the most
|
||||
# misleading failure this tool can produce.
|
||||
if [[ -n "$KIND" ]]; then
|
||||
case "$KIND" in
|
||||
header|readme|manual|template|doc|ui) ;;
|
||||
*) echo "Unknown --kind=$KIND (expected: header, readme, manual, template, doc, ui)" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
CLI="${SCRIPT_DIR}/lib/cli.js"
|
||||
DB="${AI_INDEX_DB:-${DATA_DIR}/ai_index.db}"
|
||||
|
||||
_url_var="${MY_ID}_OLLAMA_URL"
|
||||
_gen_var="${MY_ID}_OLLAMA_MODEL"
|
||||
_emb_var="${MY_ID}_OLLAMA_EMBED_MODEL"
|
||||
OLLAMA_URL="${!_url_var:-}"
|
||||
GEN_MODEL="${!_gen_var:-}"
|
||||
EMBED_MODEL="${!_emb_var:-nomic-embed-text}"
|
||||
|
||||
if [[ "$STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR AI Query Status"
|
||||
echo " Enabled: ${AI_ENABLED:-false}"
|
||||
echo " Index: $DB $([[ -f "$DB" ]] && echo "($(sqlite3 "$DB" 'SELECT COUNT(*) FROM vv_chunks;' 2>/dev/null) chunks)" || echo '(not built)')"
|
||||
echo " Ollama: ${OLLAMA_URL:-<none>}"
|
||||
echo " Generate: ${GEN_MODEL:-<unset>}"
|
||||
echo " Embed: $EMBED_MODEL"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${AI_ENABLED:-false}" != "true" ]]; then
|
||||
echo "AI_ENABLED is not true — AI features are off" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ -z "$QUERY" ]] && { echo "usage: ai_query.sh [--search] [--section=NAME] [--kind=KIND] \"your question\"" >&2; exit 1; }
|
||||
[[ -f "$DB" ]] || { error "No index at $DB — run AI/ai_index.sh first"; exit 1; }
|
||||
[[ -f "$CLI" ]] || { error "missing $CLI"; exit 1; }
|
||||
command -v node >/dev/null 2>&1 || { error "node not found"; exit 1; }
|
||||
[[ -z "$OLLAMA_URL" ]] && { error "${MY_ID}_OLLAMA_URL is empty"; exit 1; }
|
||||
|
||||
if ! curl -sf --max-time "${AI_CONNECT_TIMEOUT:-5}" "${OLLAMA_URL%/}/api/tags" >/dev/null 2>&1; then
|
||||
error "Ollama unreachable at $OLLAMA_URL"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_args=("--db=${DB}" "--url=${OLLAMA_URL}" "--query=${QUERY}")
|
||||
[[ -n "$SECTION" ]] && _args+=("--section=${SECTION}")
|
||||
[[ -n "$KIND" ]] && _args+=("--kind=${KIND}")
|
||||
[[ "$JSON" == true ]] && _args+=(--json)
|
||||
|
||||
if [[ "$SEARCH_ONLY" == true ]]; then
|
||||
node --no-warnings "$CLI" search "${_args[@]}" \
|
||||
"--model=${EMBED_MODEL}" \
|
||||
"--k=${AI_SEARCH_K:-8}" "--per-file=${AI_SEARCH_PER_FILE:-3}"
|
||||
else
|
||||
[[ -z "$GEN_MODEL" ]] && { error "${MY_ID}_OLLAMA_MODEL is empty — needed for generation"; exit 1; }
|
||||
# Token accounting. Passed in rather than re-read in node, so the conf stays the shell's job
|
||||
# and cli.js keeps taking everything it needs as arguments. Omitting either flag simply
|
||||
# skips the row — the CLI must still work when called by hand outside this wrapper.
|
||||
node --no-warnings "$CLI" ask "${_args[@]}" \
|
||||
"--model=${GEN_MODEL}" "--embed-model=${EMBED_MODEL}" \
|
||||
"--k=${AI_SEARCH_K:-8}" "--per-file=${AI_SEARCH_PER_FILE:-3}" \
|
||||
"--timeout=$(( ${AI_REQUEST_TIMEOUT:-240} * 1000 ))" \
|
||||
"--token-db=${AI_TOKEN_DB:-}" "--token-host=$(echo "$MY_ID" | tr '[:upper:]' '[:lower:]')"
|
||||
fi
|
||||
Executable
+236
@@ -0,0 +1,236 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= AI Token Ledger Sync ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Pulls each partner's AI token ledger into a RAM cache at /tmp/.cache/vv/ai/, so the AI tab
|
||||
# can report usage for the whole fleet instead of only the host the browser happens to be on.
|
||||
#
|
||||
# Each host records its own turns to data/ai_token_history.db and nothing syncs that file, so
|
||||
# without this a host can only ever see its own totals. The tab is careful to say "not collected
|
||||
# here" rather than 0 for a partner it cannot see; this script is what turns that into a number.
|
||||
#
|
||||
# Same trick as conf_sync.sh, and deliberately so — resolve the partner over Tailscale, scp one
|
||||
# small file into a tmpfs cache, let the reader treat a missing file as "unknown".
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Gates — PARTNERSHIP_ENABLED, AI_ENABLED, AI_TOKEN_SYNC_ENABLED
|
||||
# 2. Per partner:
|
||||
# a. Resolve their Tailscale IP
|
||||
# b. Resolve their SCRIPTS_DIR from their varaverk.cfg (they may be in appdata mode)
|
||||
# c. scp their data/ai_token_history.db → $AI_TOKEN_CACHE_DIR/<slot>.tokens.db
|
||||
#
|
||||
# Pull only, no push. conf_sync.sh pushes as well because a partner may be unable to reach us
|
||||
# and still needs our credentials; nothing here is needed by anyone else, and a reader that
|
||||
# fetches its own data controls its own freshness rather than depending on the partner's cron.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A missing file means unknown, never zero.
|
||||
# The whole point of the tab reporting "not collected here" is that it is a different claim
|
||||
# from "this partner spent nothing". If a partner is dark, unreachable or has never run a turn,
|
||||
# there is simply no cache file, and the reader is expected to say so rather than render a 0
|
||||
# that looks like a measurement.
|
||||
#
|
||||
# The reader pulls; nobody pushes.
|
||||
# conf_sync.sh pushes as well, because a partner that cannot reach us still needs our
|
||||
# credentials. Nothing here is needed by anyone else, so a host that wants fleet totals fetches
|
||||
# them and owns its own freshness instead of depending on someone else's cron having run.
|
||||
#
|
||||
# RAM, not flash.
|
||||
# The cache lands in tmpfs. It is a copy of a file that already exists on the partner and is
|
||||
# rebuilt on the next pass, so writing it to flash would cost wear for something that is never
|
||||
# worth surviving a reboot.
|
||||
#
|
||||
# Same shape as conf_sync.sh, deliberately.
|
||||
# Resolve over Tailscale, scp one small file into a tmpfs cache, let a missing file mean
|
||||
# unknown. A second transport pattern for a second small file would be a second set of
|
||||
# failure modes to learn.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# An unreachable partner is not a failure.
|
||||
# HOST2 is expected to be down for long stretches during onboarding. A warn every four
|
||||
# hours would train the operator to ignore this script's output, and the AI diagnostic
|
||||
# path treats every log WARN as actionable. Unresolvable partners are counted and
|
||||
# reported once at info level; only a partner that resolves and then fails to transfer
|
||||
# is treated as an error.
|
||||
#
|
||||
# The cache is never written directly.
|
||||
# scp lands on a .part file that is renamed into place, so a transfer interrupted halfway
|
||||
# cannot leave the reader parsing half a ledger. A truncated final row would be skipped by
|
||||
# the field-count check on the PHP side, but a torn file should not reach it at all.
|
||||
#
|
||||
# Nothing is ever written back to the partner.
|
||||
# This script only reads. A bug here cannot corrupt a partner's accounting.
|
||||
#
|
||||
# The cache is tmpfs and deliberately not preserved.
|
||||
# Unlike the conf cache there is no save/restore pair. Stale counters are worse than
|
||||
# absent ones: absent reads as "not collected here", stale reads as fact.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# AI_ENABLED Whole AI subsystem gate
|
||||
# AI_TOKEN_SYNC_ENABLED This script's own toggle (default: true)
|
||||
# PARTNERSHIP_ENABLED Checked via require_partnership()
|
||||
# SSH_KEY Key used for all partner ssh/scp operations
|
||||
#
|
||||
# load_config.sh
|
||||
#
|
||||
# AI_TOKEN_CACHE_DIR tmpfs directory the tab reads partner ledgers from
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST* — hostnames used to build the partner list via detect_hosts()
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ai_token_sync.sh Pull every reachable partner's ledger
|
||||
# ai_token_sync.sh --dry-run Report what would be pulled, transfer nothing
|
||||
# ai_token_sync.sh --log Verbose output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
require_partnership
|
||||
|
||||
if [[ "${AI_ENABLED:-false}" != true ]]; then
|
||||
log "AI_ENABLED=false — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${AI_TOKEN_SYNC_ENABLED:-true}" == false ]]; then
|
||||
log "AI_TOKEN_SYNC_ENABLED=false — skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CACHE_DIR="$AI_TOKEN_CACHE_DIR"
|
||||
SSH_TIMEOUT=10
|
||||
|
||||
# Mirrors conf_sync.sh — the remote may be in appdata storage mode, so its ledger is not
|
||||
# necessarily under /boot.
|
||||
_remote_scripts_dir() {
|
||||
local ip="$1"
|
||||
local cfg line sd
|
||||
cfg=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${ip}" "cat /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null" 2>/dev/null) || true
|
||||
while IFS= read -r line; do
|
||||
[[ "$line" == SCRIPTS_DIR=* ]] || continue
|
||||
sd="${line#SCRIPTS_DIR=}"; sd="${sd//\"/}"; sd="${sd//\'/}"
|
||||
echo "$sd"; return
|
||||
done <<< "$cfg"
|
||||
echo "/boot/config/plugins/varaverk"
|
||||
}
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$CACHE_DIR" && chmod 755 "$CACHE_DIR"
|
||||
fi
|
||||
|
||||
PULLED=0
|
||||
OFFLINE=0
|
||||
FAILED=0
|
||||
|
||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
|
||||
partner_host="${!host_var}"
|
||||
[[ -z "$partner_host" ]] && continue
|
||||
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
|
||||
|
||||
partner_slot="${host_var,,}"
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$partner_ip" ]]; then
|
||||
log "$partner_host — unresolvable, leaving any cached ledger as-is"
|
||||
(( OFFLINE++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# Liveness and ledger presence are separate questions, probed separately on purpose.
|
||||
# Tailscale hands back an IP for a peer that is registered but powered off, so resolving
|
||||
# proves nothing. And a single `ssh test -f` answers both questions at once: it fails
|
||||
# identically whether the host is down or the file is simply absent. Treating that one
|
||||
# failure as "no ledger" would delete a perfectly good cached copy every time the partner
|
||||
# blinked — turning "synced 3h ago" into "not collected here" on a transient.
|
||||
if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" true 2>/dev/null; then
|
||||
log "$partner_host — not answering, keeping any cached ledger as-is"
|
||||
(( OFFLINE++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
remote_sd=$(_remote_scripts_dir "$partner_ip")
|
||||
remote_db="${remote_sd}/data/ai_token_history.db"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull $partner_host:$remote_db → $CACHE_DIR/${partner_slot}.tokens.db"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Reachable, but nothing recorded there — a partner with AI off, or one that has simply
|
||||
# never been asked anything. Now that liveness is established this is a real answer, so the
|
||||
# stale copy goes: the tab should say "not collected here", not quote a number from before.
|
||||
if ! timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}" "[[ -f '$remote_db' ]]" 2>/dev/null; then
|
||||
log "$partner_host — reachable, but no ledger there yet"
|
||||
rm -f "$CACHE_DIR/${partner_slot}.tokens.db"
|
||||
(( OFFLINE++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
"root@${partner_ip}:${remote_db}" \
|
||||
"$CACHE_DIR/${partner_slot}.tokens.db.part" 2>/dev/null \
|
||||
&& mv -f "$CACHE_DIR/${partner_slot}.tokens.db.part" "$CACHE_DIR/${partner_slot}.tokens.db"; then
|
||||
chmod 644 "$CACHE_DIR/${partner_slot}.tokens.db" 2>/dev/null
|
||||
echo "Pulled ${partner_slot} ledger from $partner_host ✅"
|
||||
(( PULLED++ ))
|
||||
else
|
||||
rm -f "$CACHE_DIR/${partner_slot}.tokens.db.part"
|
||||
warn "Could not pull ${partner_slot} ledger from $partner_host"
|
||||
(( FAILED++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
# Zero counts are omitted rather than printed. ${VAR:+...} keeps "0" because it is a non-empty
|
||||
# string, and a summary that always ends "0 failed" is what teaches you to stop reading it.
|
||||
_summary="AI token sync complete — pulled $PULLED"
|
||||
(( OFFLINE > 0 )) && _summary+=", $OFFLINE unavailable"
|
||||
(( FAILED > 0 )) && _summary+=", $FAILED failed"
|
||||
info "$_summary"
|
||||
|
||||
# Only a partner that answered and then failed the transfer is worth an exit code. An absent
|
||||
# partner is the normal state whenever a partner is not yet onboarded.
|
||||
[[ "$FAILED" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
'use strict';
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// Chunker — turns repo files into retrieval units.
|
||||
//
|
||||
// The whole point of the header audit is that chunk boundaries are deterministic here. Bash
|
||||
// scripts split on their six section names, markdown on its headings, conf templates on their
|
||||
// ━━━ section rules. Nothing is split on a fixed token window, so no chunk ever contains half
|
||||
// of one idea and half of another.
|
||||
//
|
||||
// Every chunk carries its section name as its own field, because that is the metadata that
|
||||
// lets retrieval filter by question shape before it ever computes similarity.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const BASH_SECTIONS = [
|
||||
'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES',
|
||||
'OPERATIONAL SAFEGUARDS', 'CONFIGURATION', 'RUNTIME MODES',
|
||||
];
|
||||
|
||||
// PHP headers reuse the first three names deliberately, then diverge per layer.
|
||||
const PHP_SECTIONS = [
|
||||
'PURPOSE', 'OPERATIONAL MODEL', 'DESIGN PRINCIPLES', 'OPERATIONAL SAFEGUARDS',
|
||||
'STATUS', 'EXPORTS', 'REQUEST CONTRACT', 'SIDE EFFECTS', 'RENDERS', 'DEPENDS ON',
|
||||
'CONFIGURATION',
|
||||
];
|
||||
|
||||
const MIN_CHARS = 40; // below this a chunk carries no retrievable meaning
|
||||
const MAX_CHARS = 6000; // above this, split on blank lines — protects the embed window
|
||||
|
||||
// Banner rules and box-drawing art are everywhere in this repo's headers. They carry no
|
||||
// meaning to embed, and a chunk that is mostly rule characters is pure noise in the index.
|
||||
// Measure a chunk by what is left after the decoration is removed, not by raw length.
|
||||
function meaningful(s) {
|
||||
return s.replace(/[═─━=_#\/*\s|+.-]/g, '').length;
|
||||
}
|
||||
const MIN_MEANINGFUL = 30;
|
||||
|
||||
function isRealHeading(h) {
|
||||
return !!h && /[A-Za-z0-9]/.test(h.replace(/[═─━=_]/g, ''));
|
||||
}
|
||||
|
||||
function stripPrefix(line, prefix) {
|
||||
// '# text' -> 'text' '// text' -> 'text'
|
||||
const re = new RegExp('^\\s*' + prefix + '\\s?');
|
||||
return line.replace(re, '');
|
||||
}
|
||||
|
||||
// ── Comment-header sectioning, shared by bash (#) and PHP (//) ────────────────────────────────
|
||||
function sectionsFromCommentHeader(text, prefix, names) {
|
||||
const lines = text.split('\n');
|
||||
const nameSet = new Set(names);
|
||||
const found = [];
|
||||
|
||||
// headerEnd matters as much as the section starts. The last section (RUNTIME MODES in bash,
|
||||
// DEPENDS ON in a page) would otherwise run to EOF and sweep up every unrelated comment in
|
||||
// the file — scheduler.php alone contributed an 11k-char chunk of unrelated inline comments.
|
||||
let headerEnd = lines.length;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const raw = lines[i];
|
||||
if (!new RegExp('^\\s*' + prefix).test(raw)) {
|
||||
// Header block ends at the first non-comment, non-blank line past the shebang.
|
||||
// '<?php' and '?>' bracket a PHP header block and are not the end of it.
|
||||
const t = raw.trim();
|
||||
if (t !== '' && !/^#!/.test(t) && t !== '<?php' && t !== '?>' && found.length) {
|
||||
headerEnd = i;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const inner = stripPrefix(raw, prefix).trim();
|
||||
if (nameSet.has(inner)) found.push({ name: inner, start: i });
|
||||
}
|
||||
|
||||
const out = [];
|
||||
for (let k = 0; k < found.length; k++) {
|
||||
const start = found[k].start + 1;
|
||||
const end = k + 1 < found.length ? found[k + 1].start : headerEnd;
|
||||
const body = lines.slice(start, end)
|
||||
.filter(l => new RegExp('^\\s*' + prefix).test(l))
|
||||
.map(l => stripPrefix(l, prefix))
|
||||
// drop pure separator rules (════, ────, ━━━) — they carry no meaning
|
||||
.filter(l => !/^[\s═─━=_-]*$/.test(l) || l.trim() === '')
|
||||
.join('\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
if (meaningful(body) >= MIN_MEANINGFUL) {
|
||||
for (const p of splitNamedParagraphs(body))
|
||||
out.push({ section: found[k].name, title: p.title, content: p.content });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Named-paragraph sub-chunking ──────────────────────────────────────────────────────────────
|
||||
// The header convention writes safeguards and principles as named paragraphs: an unindented
|
||||
// title line followed by an indented body. Embedding a whole section as one unit dilutes them —
|
||||
// rsync.sh's OPERATIONAL SAFEGUARDS holds eight distinct guarantees in 2.8k chars, and a query
|
||||
// about one of them scored below unrelated chunks because the other seven dominated the vector.
|
||||
// Splitting on the title lines is what makes a specific question find a specific answer.
|
||||
//
|
||||
// The section name is carried onto every sub-chunk, so section routing still works; the
|
||||
// paragraph title becomes the chunk's heading.
|
||||
function splitNamedParagraphs(body) {
|
||||
const lines = body.split('\n');
|
||||
const marks = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const l = lines[i];
|
||||
if (!l.trim()) continue;
|
||||
if (/^\s/.test(l)) continue; // indented => body, not a title
|
||||
if (/^[-*•]/.test(l.trim())) continue; // list item, not a title
|
||||
if (l.trim().length > 80) continue; // a long line is prose, not a heading
|
||||
if (/[.:;,]$/.test(l.trim())) continue; // ends like a sentence
|
||||
|
||||
// The decisive signal: a real title is followed by an indented body. Wrapped prose is
|
||||
// followed by more unindented prose. Without this check, any short line in a paragraph
|
||||
// that happened to wrap became a spurious chunk boundary mid-sentence.
|
||||
let j = i + 1;
|
||||
while (j < lines.length && !lines[j].trim()) j++;
|
||||
if (j >= lines.length || !/^\s+\S/.test(lines[j])) continue;
|
||||
|
||||
marks.push(i);
|
||||
}
|
||||
// Fewer than two titles means this section is not written as named paragraphs — keep it whole.
|
||||
if (marks.length < 2) return [{ title: null, content: body }];
|
||||
|
||||
const out = [];
|
||||
if (marks[0] > 0) {
|
||||
const pre = lines.slice(0, marks[0]).join('\n').trim();
|
||||
if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ title: null, content: pre });
|
||||
}
|
||||
for (let k = 0; k < marks.length; k++) {
|
||||
const start = marks[k];
|
||||
const end = k + 1 < marks.length ? marks[k + 1] : lines.length;
|
||||
const title = lines[start].trim();
|
||||
const content = lines.slice(start, end).join('\n').trim();
|
||||
if (meaningful(content) >= MIN_MEANINGFUL) out.push({ title, content });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Markdown: split on ## headings, keep the heading with its body ─────────────────────────────
|
||||
function sectionsFromMarkdown(text) {
|
||||
const lines = text.split('\n');
|
||||
const marks = [];
|
||||
let fence = false;
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (/^\s*```/.test(lines[i])) { fence = !fence; continue; }
|
||||
if (fence) continue;
|
||||
// A heading whose text is nothing but rule characters is a banner, not a section. The
|
||||
// house style opens a document with three of them —
|
||||
// # ━━━━━━━━
|
||||
// # 🏠 VARAVERK
|
||||
// # ━━━━━━━━
|
||||
// — and treating each as a boundary split the title into a section of its own, too small
|
||||
// to survive, then gave the paragraph that actually defines the project a chunk headed by
|
||||
// the rule beneath it: no heading, and content opening with 75 identical glyphs. That is
|
||||
// why "what is Varaverk" returned five script PURPOSE headers and never README.md, which
|
||||
// has been indexed the whole time. 35 of the 40 markdown files here open this way.
|
||||
//
|
||||
// Whole-string test, so an ordinary heading containing a dash is unaffected — "Set-up"
|
||||
// does not reduce to empty, and a line of dashes does.
|
||||
if (!/^#{1,3}\s+\S/.test(lines[i])) continue;
|
||||
if (lines[i].replace(/^#+\s*/, '').replace(/[━─═=~_*\-\s]+/gu, '') === '') continue;
|
||||
marks.push(i);
|
||||
}
|
||||
if (!marks.length) return [{ heading: null, content: text.trim() }];
|
||||
|
||||
const out = [];
|
||||
// preamble before the first heading
|
||||
if (marks[0] > 0) {
|
||||
const pre = lines.slice(0, marks[0]).join('\n').trim();
|
||||
if (meaningful(pre) >= MIN_MEANINGFUL) out.push({ heading: null, content: pre });
|
||||
}
|
||||
for (let k = 0; k < marks.length; k++) {
|
||||
const start = marks[k];
|
||||
const end = k + 1 < marks.length ? marks[k + 1] : lines.length;
|
||||
const heading = lines[start].replace(/^#+\s*/, '').replace(/[━─═]+/g, '').trim();
|
||||
const content = lines.slice(start, end).join('\n').trim();
|
||||
if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: isRealHeading(heading) ? heading : null, content });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Conf templates: split on the ━━━ / ── section rules ───────────────────────────────────────
|
||||
function sectionsFromConfTemplate(text) {
|
||||
const lines = text.split('\n');
|
||||
const marks = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const m = lines[i].match(/^#\s*[━─]{2,}\s*(.+?)\s*[━─]{2,}\s*$/);
|
||||
if (m && isRealHeading(m[1])) marks.push({ i, name: m[1].trim() });
|
||||
}
|
||||
if (!marks.length) return [];
|
||||
|
||||
const out = [];
|
||||
for (let k = 0; k < marks.length; k++) {
|
||||
const start = marks[k].i;
|
||||
const end = k + 1 < marks.length ? marks[k + 1].i : lines.length;
|
||||
const content = lines.slice(start, end).join('\n').replace(/\n{3,}/g, '\n\n').trim();
|
||||
if (meaningful(content) >= MIN_MEANINGFUL) out.push({ heading: marks[k].name, content });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Oversized chunks split on blank lines rather than mid-sentence.
|
||||
function capSize(chunks) {
|
||||
const out = [];
|
||||
for (const c of chunks) {
|
||||
if (c.content.length <= MAX_CHARS) { out.push(c); continue; }
|
||||
const paras = c.content.split(/\n\s*\n/);
|
||||
let buf = [], len = 0, part = 1;
|
||||
const flush = () => {
|
||||
if (!buf.length) return;
|
||||
out.push({ ...c, content: buf.join('\n\n'), part: part++ });
|
||||
buf = []; len = 0;
|
||||
};
|
||||
for (const p of paras) {
|
||||
if (len + p.length > MAX_CHARS && buf.length) flush();
|
||||
buf.push(p); len += p.length + 2;
|
||||
}
|
||||
flush();
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function classify(rel) {
|
||||
const base = path.basename(rel);
|
||||
if (rel.startsWith('Deployment/') && rel.endsWith('.template')) return 'template';
|
||||
// WebGUI page docs, written for whoever is using the tab rather than maintaining it. Their
|
||||
// own kind because every other kind here answers a maintainer's question: an operator asking
|
||||
// "how do I stop this" needs the click path, and a corpus that is three-quarters script
|
||||
// headers will otherwise always answer in conf edits. Matched on the folder, not the
|
||||
// filename, so these can be named whatever reads best.
|
||||
if (rel.startsWith('Plugin/unraid/pages/readme/') && base.endsWith('.md')) return 'ui';
|
||||
if (base.endsWith('.md')) {
|
||||
if (base.startsWith('Manual')) return 'manual';
|
||||
if (base.startsWith('README') || base === 'README.md') return 'readme';
|
||||
return 'doc';
|
||||
}
|
||||
if (base.endsWith('.sh')) return 'header';
|
||||
if (base.endsWith('.php')) return 'header';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
function chunkFile(absPath, rel) {
|
||||
const text = fs.readFileSync(absPath, 'utf8');
|
||||
const kind = classify(rel);
|
||||
let raw = [];
|
||||
|
||||
if (kind === 'header' && rel.endsWith('.sh')) {
|
||||
raw = sectionsFromCommentHeader(text, '#', BASH_SECTIONS)
|
||||
.map(s => ({ section: s.section, heading: s.title || null, content: s.content }));
|
||||
} else if (kind === 'header' && rel.endsWith('.php')) {
|
||||
raw = sectionsFromCommentHeader(text, '//', PHP_SECTIONS)
|
||||
.map(s => ({ section: s.section, heading: s.title || null, content: s.content }));
|
||||
} else if (kind === 'template') {
|
||||
raw = sectionsFromConfTemplate(text)
|
||||
.map(s => ({ section: null, heading: s.heading, content: s.content }));
|
||||
} else if (kind === 'readme' || kind === 'manual' || kind === 'doc' || kind === 'ui') {
|
||||
raw = sectionsFromMarkdown(text)
|
||||
.map(s => ({ section: null, heading: s.heading, content: s.content }));
|
||||
}
|
||||
|
||||
return capSize(raw).map(c => ({
|
||||
path: rel,
|
||||
kind,
|
||||
section: c.section || null,
|
||||
heading: c.heading || null,
|
||||
part: c.part || null,
|
||||
content: c.content,
|
||||
}));
|
||||
}
|
||||
|
||||
module.exports = { chunkFile, classify, BASH_SECTIONS, PHP_SECTIONS };
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
'use strict';
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// CLI bridge — the thin layer the bash entry points call.
|
||||
//
|
||||
// The bash scripts own configuration, gating, locking and logging, exactly as they do for every
|
||||
// other Varaverk job. This file owns only the work that is genuinely awkward in bash: float
|
||||
// vector math and SQLite BLOBs. That split follows the existing api_cache_writer.sh precedent —
|
||||
// a bash shim in front of the language that fits the task.
|
||||
//
|
||||
// Every value arrives as an argument or an environment variable read by the caller. This file
|
||||
// never reads a conf file itself, so there is exactly one place that decides what the settings
|
||||
// are.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const fs = require('fs');
|
||||
const { buildIndex } = require('./index.js');
|
||||
const { search } = require('./search.js');
|
||||
|
||||
function arg(name, dflt) {
|
||||
const p = `--${name}=`;
|
||||
const hit = process.argv.find(a => a.startsWith(p));
|
||||
return hit ? hit.slice(p.length) : dflt;
|
||||
}
|
||||
function flag(name) {
|
||||
return process.argv.includes(`--${name}`);
|
||||
}
|
||||
|
||||
// Token accounting. Writes the same row shape as the WebGUI worker into the same file — one
|
||||
// ledger for both paths, or the totals quietly come to mean "whatever the tab happened to do".
|
||||
// Skipped silently when the caller passes neither flag, because cli.js has to stay runnable by
|
||||
// hand. Best-effort: a failed append must never cost a caller an answer it already has.
|
||||
//
|
||||
// Trimming is deliberately not done here. The PHP side prunes on write, and duplicating a
|
||||
// read-modify-write of the whole file in a second language is how the two drift apart.
|
||||
function recordTokens(profile, prompt, completion, tokS) {
|
||||
const db = arg('token-db', ''), host = arg('token-host', '');
|
||||
if (!db || !host || (prompt <= 0 && completion <= 0)) return;
|
||||
const d = new Date();
|
||||
const p2 = n => String(n).padStart(2, '0');
|
||||
const row = [
|
||||
`${d.getFullYear()}-${p2(d.getMonth() + 1)}-${p2(d.getDate())}`,
|
||||
`${p2(d.getHours())}:${p2(d.getMinutes())}:${p2(d.getSeconds())}`,
|
||||
host, profile, 'cli', prompt, completion,
|
||||
tokS === null ? '' : tokS.toFixed(1),
|
||||
].join('|') + '\n';
|
||||
try { fs.appendFileSync(db, row); } catch { /* accounting is not the answer */ }
|
||||
}
|
||||
|
||||
function fail(msg, code = 1) {
|
||||
console.error(msg);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
async function cmdIndex() {
|
||||
const root = arg('root');
|
||||
const db = arg('db');
|
||||
const url = arg('url');
|
||||
const model = arg('model', 'nomic-embed-text');
|
||||
if (!root || !db || !url) fail('index: --root, --db and --url are required');
|
||||
|
||||
const quiet = flag('quiet');
|
||||
let stats;
|
||||
try {
|
||||
stats = await buildIndex({
|
||||
root, dbPath: db, url, model,
|
||||
batch: parseInt(arg('batch', '32'), 10),
|
||||
timeout: parseInt(arg('timeout', '120000'), 10),
|
||||
force: flag('force'),
|
||||
dryRun: flag('dry-run'),
|
||||
onProgress: p => {
|
||||
if (p.error) console.error(`embed batch failed: ${p.error}`);
|
||||
else if (!quiet && p.done % 320 === 0) console.log(` embedded ${p.done}/${p.total}`);
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`index failed: ${e.message}`, 2);
|
||||
}
|
||||
|
||||
if (flag('json')) { console.log(JSON.stringify(stats)); return; }
|
||||
if (stats.dryRun) {
|
||||
console.log(`DRY RUN — ${stats.files} file(s) would be indexed, ${stats.chunks} chunk(s) embedded`);
|
||||
console.log(` ${stats.skipped} unchanged, ${stats.removed} stale entr(ies) would be dropped`);
|
||||
return;
|
||||
}
|
||||
console.log(`indexed ${stats.files} file(s), ${stats.chunks} chunk(s) embedded`);
|
||||
console.log(` ${stats.skipped} unchanged, ${stats.removed} removed, ${stats.failed} failed`);
|
||||
console.log(` index now holds ${stats.total} chunk(s)`);
|
||||
// A partial index is usable but not complete — say so in the exit code so a caller can act.
|
||||
if (stats.failed) process.exit(3);
|
||||
}
|
||||
|
||||
async function cmdSearch() {
|
||||
const db = arg('db');
|
||||
const url = arg('url');
|
||||
const model = arg('model', 'nomic-embed-text');
|
||||
const q = arg('query');
|
||||
if (!db || !url || !q) fail('search: --db, --url and --query are required');
|
||||
|
||||
let r;
|
||||
try {
|
||||
r = await search({
|
||||
dbPath: db, url, embedModel: model, query: q,
|
||||
k: parseInt(arg('k', '8'), 10),
|
||||
perFile: parseInt(arg('per-file', '3'), 10),
|
||||
section: arg('section', null),
|
||||
kind: arg('kind', null),
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`search failed: ${e.message}`, 2);
|
||||
}
|
||||
|
||||
if (flag('json')) { console.log(JSON.stringify(r)); return; }
|
||||
if (!r.results.length) { console.log('no matches'); return; }
|
||||
if (r.intents.length) console.log(`intent: ${r.intents.join(', ')}\n`);
|
||||
for (const x of r.results) {
|
||||
const label = x.heading || x.section || '-';
|
||||
console.log(`── ${x.score.toFixed(3)} ${x.path} [${x.section || x.kind}] ${label}`);
|
||||
console.log(x.content.split('\n').map(l => ' ' + l).join('\n'));
|
||||
console.log('');
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieval + generation. The prompt is built here so the context block and the instructions
|
||||
// stay in one reviewable place.
|
||||
async function cmdAsk() {
|
||||
const db = arg('db');
|
||||
const url = arg('url');
|
||||
const embed = arg('embed-model', 'nomic-embed-text');
|
||||
const gen = arg('model');
|
||||
const q = arg('query');
|
||||
const timeout = parseInt(arg('timeout', '240000'), 10);
|
||||
if (!db || !url || !gen || !q) fail('ask: --db, --url, --model and --query are required');
|
||||
|
||||
let r;
|
||||
try {
|
||||
// section and kind must be forwarded here too. They were not, so both filters worked
|
||||
// under `search` and were silently ignored under `ask` — the documented
|
||||
// --section=CONFIGURATION usage retrieved from the whole index and the answer looked
|
||||
// plausible, which is the worst way for a filter to fail.
|
||||
r = await search({
|
||||
dbPath: db, url, embedModel: embed, query: q,
|
||||
k: parseInt(arg('k', '6'), 10), perFile: parseInt(arg('per-file', '2'), 10),
|
||||
section: arg('section', null),
|
||||
kind: arg('kind', null),
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`retrieval failed: ${e.message}`, 2);
|
||||
}
|
||||
if (!r.results.length) fail('no relevant context found in the index', 4);
|
||||
|
||||
const context = r.results.map((x, i) => {
|
||||
const label = [x.path, x.section, x.heading].filter(Boolean).join(' › ');
|
||||
return `[${i + 1}] ${label}\n${x.content}`;
|
||||
}).join('\n\n');
|
||||
|
||||
const prompt =
|
||||
`You are answering questions about Varaverk, a two-server self-healing home media ecosystem.
|
||||
|
||||
Answer ONLY from the context below. If the context does not contain the answer, say so plainly
|
||||
and name what is missing — do not fill the gap from general knowledge about Linux, Docker or
|
||||
rsync, because this system's conventions are frequently not the conventional ones.
|
||||
|
||||
Cite the source of each claim as [n]. Be concise and concrete.
|
||||
|
||||
CONTEXT
|
||||
${context}
|
||||
|
||||
QUESTION
|
||||
${q}
|
||||
|
||||
ANSWER`;
|
||||
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(`${url.replace(/\/$/, '')}/api/generate`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model: gen, prompt, stream: false,
|
||||
options: { temperature: 0.2, num_ctx: 8192 },
|
||||
}),
|
||||
signal: AbortSignal.timeout(timeout),
|
||||
});
|
||||
} catch (e) {
|
||||
fail(`generation failed: ${e.message}`, 2);
|
||||
}
|
||||
if (!res.ok) fail(`generation HTTP ${res.status}`, 2);
|
||||
const j = await res.json();
|
||||
|
||||
// 'varaverk' rather than a CLI-specific name: this path retrieves and cites, so it is the
|
||||
// same kind of turn the tab's default profile runs, and the two should aggregate together.
|
||||
recordTokens('varaverk', j.prompt_eval_count || 0, j.eval_count || 0,
|
||||
j.eval_duration > 0 ? (j.eval_count / (j.eval_duration / 1e9)) : null);
|
||||
|
||||
if (flag('json')) {
|
||||
console.log(JSON.stringify({ answer: j.response, sources: r.results.map(x => ({ path: x.path, section: x.section, heading: x.heading, score: x.score })) }));
|
||||
return;
|
||||
}
|
||||
console.log((j.response || '').trim());
|
||||
console.log('\nSources:');
|
||||
r.results.forEach((x, i) => {
|
||||
console.log(` [${i + 1}] ${[x.path, x.section, x.heading].filter(Boolean).join(' › ')}`);
|
||||
});
|
||||
}
|
||||
|
||||
const cmd = process.argv[2];
|
||||
const table = { index: cmdIndex, search: cmdSearch, ask: cmdAsk };
|
||||
if (!table[cmd]) fail(`usage: cli.js <index|search|ask> [--flags]`);
|
||||
table[cmd]().catch(e => fail(e.message, 2));
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
'use strict';
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// Indexer — chunk the repo, embed each chunk, store vectors in SQLite.
|
||||
//
|
||||
// Incremental by file mtime: a file whose mtime has not moved since its last index is skipped
|
||||
// entirely, so a routine re-index costs seconds rather than re-embedding the whole corpus.
|
||||
//
|
||||
// Vectors are stored as raw little-endian float32 BLOBs. nomic-embed-text returns L2-normalised
|
||||
// vectors, so cosine similarity is a plain dot product at query time — no normalising, no
|
||||
// magnitude cache. PHP can read the same blobs with unpack('f*', $blob) when the UI needs them.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
const { chunkFile, classify } = require('./chunk.js');
|
||||
|
||||
const SCHEMA = `
|
||||
CREATE TABLE IF NOT EXISTS vv_files (
|
||||
path TEXT PRIMARY KEY,
|
||||
mtime INTEGER NOT NULL,
|
||||
chunks INTEGER NOT NULL,
|
||||
indexed INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS vv_chunks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
section TEXT,
|
||||
heading TEXT,
|
||||
part INTEGER,
|
||||
content TEXT NOT NULL,
|
||||
vector BLOB NOT NULL,
|
||||
indexed INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_path ON vv_chunks(path);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_section ON vv_chunks(section);
|
||||
CREATE INDEX IF NOT EXISTS idx_chunks_kind ON vv_chunks(kind);
|
||||
CREATE TABLE IF NOT EXISTS vv_meta (k TEXT PRIMARY KEY, v TEXT);
|
||||
`;
|
||||
|
||||
function openDb(dbPath) {
|
||||
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
||||
const db = new DatabaseSync(dbPath);
|
||||
db.exec('PRAGMA journal_mode = WAL;');
|
||||
db.exec('PRAGMA synchronous = NORMAL;');
|
||||
db.exec(SCHEMA);
|
||||
return db;
|
||||
}
|
||||
|
||||
// Only ever index what git tracks. Configurations/, State_Files/ and data/ are gitignored, which
|
||||
// is what makes it structurally impossible for a credential to reach the index — the files that
|
||||
// hold them were never in the repo. Do not replace this with a filesystem walk.
|
||||
function trackedFiles(root) {
|
||||
return execSync('git ls-files', { cwd: root, maxBuffer: 1 << 26 })
|
||||
.toString().trim().split('\n')
|
||||
.filter(Boolean)
|
||||
.filter(f => classify(f) !== 'other');
|
||||
}
|
||||
|
||||
async function embedBatch(url, model, inputs, timeoutMs) {
|
||||
const ctl = AbortSignal.timeout(timeoutMs);
|
||||
const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, input: inputs }),
|
||||
signal: ctl,
|
||||
});
|
||||
if (!res.ok) throw new Error(`embed HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`);
|
||||
const j = await res.json();
|
||||
if (!j.embeddings || j.embeddings.length !== inputs.length)
|
||||
throw new Error(`embed returned ${j.embeddings ? j.embeddings.length : 0} of ${inputs.length}`);
|
||||
return j.embeddings;
|
||||
}
|
||||
|
||||
function toBlob(vec) {
|
||||
return Buffer.from(Float32Array.from(vec).buffer);
|
||||
}
|
||||
|
||||
async function buildIndex(opts) {
|
||||
const {
|
||||
root, dbPath, url, model,
|
||||
batch = 32, timeout = 120000, force = false, dryRun = false,
|
||||
onProgress = () => {},
|
||||
} = opts;
|
||||
|
||||
const db = dryRun ? null : openDb(dbPath);
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
|
||||
const known = new Map();
|
||||
if (db) for (const r of db.prepare('SELECT path, mtime FROM vv_files').all()) known.set(r.path, r.mtime);
|
||||
|
||||
const files = trackedFiles(root);
|
||||
const present = new Set(files);
|
||||
|
||||
// Files that left the repo must leave the index with them.
|
||||
let removed = 0;
|
||||
if (db && !force) {
|
||||
for (const p of known.keys()) {
|
||||
if (!present.has(p)) {
|
||||
db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(p);
|
||||
db.prepare('DELETE FROM vv_files WHERE path = ?').run(p);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (db && force) { db.exec('DELETE FROM vv_chunks; DELETE FROM vv_files;'); }
|
||||
|
||||
// ── Collect the chunks that actually need embedding ───────────────────────────────────────
|
||||
const pending = [];
|
||||
let skipped = 0, scanned = 0;
|
||||
|
||||
for (const rel of files) {
|
||||
const abs = path.join(root, rel);
|
||||
let st;
|
||||
try { st = fs.statSync(abs); } catch { continue; }
|
||||
const mtime = Math.floor(st.mtimeMs / 1000);
|
||||
scanned++;
|
||||
|
||||
if (!force && known.has(rel) && known.get(rel) === mtime) { skipped++; continue; }
|
||||
|
||||
let chunks = [];
|
||||
try { chunks = chunkFile(abs, rel); } catch (e) { continue; }
|
||||
// A file that yields no chunks still gets a vv_files row so it is not re-chunked every
|
||||
// run. Queued rather than written here, so every database change lands in the single
|
||||
// commit below — a run interrupted mid-embed must leave the index exactly as it was.
|
||||
pending.push({ rel, mtime, chunks });
|
||||
}
|
||||
|
||||
const totalChunks = pending.reduce((n, f) => n + f.chunks.length, 0);
|
||||
if (dryRun) {
|
||||
return { dryRun: true, scanned, skipped, removed, files: pending.length, chunks: totalChunks };
|
||||
}
|
||||
|
||||
// ── Embed in batches, write per file so an interrupted run leaves a consistent index ───────
|
||||
const flat = [];
|
||||
for (const f of pending) for (const c of f.chunks) flat.push({ f, c });
|
||||
|
||||
let done = 0, failed = 0;
|
||||
for (let i = 0; i < flat.length; i += batch) {
|
||||
const slice = flat.slice(i, i + batch);
|
||||
const inputs = slice.map(x => x.c.content);
|
||||
let vecs;
|
||||
try {
|
||||
vecs = await embedBatch(url, model, inputs, timeout);
|
||||
} catch (e) {
|
||||
failed += slice.length;
|
||||
onProgress({ done, total: flat.length, error: e.message });
|
||||
continue;
|
||||
}
|
||||
slice.forEach((x, k) => { x.c.__vec = vecs[k]; });
|
||||
done += slice.length;
|
||||
onProgress({ done, total: flat.length });
|
||||
}
|
||||
|
||||
const ins = db.prepare(
|
||||
'INSERT INTO vv_chunks (path,kind,section,heading,part,content,vector,indexed) VALUES (?,?,?,?,?,?,?,?)'
|
||||
);
|
||||
const insF = db.prepare('INSERT OR REPLACE INTO vv_files VALUES (?,?,?,?)');
|
||||
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
for (const f of pending) {
|
||||
// Nothing to index in this file at all — record it so it is not re-chunked next run.
|
||||
if (!f.chunks.length) {
|
||||
db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(f.rel);
|
||||
insF.run(f.rel, f.mtime, 0, now);
|
||||
continue;
|
||||
}
|
||||
const embedded = f.chunks.filter(c => c.__vec);
|
||||
// A file whose chunks all failed to embed keeps its previous rows and its old mtime,
|
||||
// so the next run retries it rather than recording a half-indexed file as current.
|
||||
if (!embedded.length) continue;
|
||||
db.prepare('DELETE FROM vv_chunks WHERE path = ?').run(f.rel);
|
||||
for (const c of embedded) {
|
||||
ins.run(c.path, c.kind, c.section, c.heading, c.part, c.content, toBlob(c.__vec), now);
|
||||
}
|
||||
insF.run(f.rel, f.mtime, embedded.length, now);
|
||||
}
|
||||
db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('embed_model', model);
|
||||
db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('last_index', String(now));
|
||||
db.prepare('INSERT OR REPLACE INTO vv_meta VALUES (?,?)').run('dims', '768');
|
||||
db.exec('COMMIT');
|
||||
} catch (e) {
|
||||
db.exec('ROLLBACK');
|
||||
throw e;
|
||||
}
|
||||
|
||||
const stats = {
|
||||
scanned, skipped, removed,
|
||||
files: pending.length,
|
||||
chunks: done,
|
||||
failed,
|
||||
total: db.prepare('SELECT COUNT(*) n FROM vv_chunks').get().n,
|
||||
};
|
||||
db.close();
|
||||
return stats;
|
||||
}
|
||||
|
||||
module.exports = { buildIndex, openDb, toBlob, trackedFiles };
|
||||
@@ -0,0 +1,139 @@
|
||||
'use strict';
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// Search — embed a question, score it against the index, return the best chunks.
|
||||
//
|
||||
// nomic-embed-text returns L2-normalised vectors, so cosine similarity is a plain dot product.
|
||||
// At this corpus size (~1.7k chunks x 768 dims) that is a couple of million multiply-adds —
|
||||
// under a millisecond, with no vector database and no index structure to maintain.
|
||||
//
|
||||
// Section routing is the payoff from the header audit. Every chunk knows whether it is a
|
||||
// PURPOSE, a DESIGN PRINCIPLES, an OPERATIONAL SAFEGUARDS and so on, so a question's shape can
|
||||
// steer retrieval before similarity is even considered. It is applied as a score boost rather
|
||||
// than a hard filter — intent detection is a heuristic, and a heuristic should not be able to
|
||||
// exclude the one chunk that actually holds the answer.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const { DatabaseSync } = require('node:sqlite');
|
||||
|
||||
// Question shape → the section most likely to answer it.
|
||||
const INTENTS = [
|
||||
{ section: 'OPERATIONAL SAFEGUARDS',
|
||||
re: /\b(safe|safety|guard|protect|prevent|fail|failure|abort|refuse|lock|root|timeout|dry.?run|what stops|what happens if|race|corrupt|data.?loss)\b/i },
|
||||
{ section: 'CONFIGURATION',
|
||||
re: /\b(variable|var|setting|conf|config|threshold|toggle|which key|what controls|where is .* set|default value|env)\b/i },
|
||||
{ section: 'RUNTIME MODES',
|
||||
re: /\b(flag|argument|option|--\w+|how do i run|invoke|cli|command line|status mode|usage)\b/i },
|
||||
{ section: 'DESIGN PRINCIPLES',
|
||||
re: /\b(why|rationale|reason|design|decision|deliberate|intentional|on purpose|trade.?off|chose|approach)\b/i },
|
||||
{ section: 'OPERATIONAL MODEL',
|
||||
re: /\b(how does .* work|flow|sequence|order|tier|lifecycle|state machine|when does)\b/i },
|
||||
{ section: 'EXPORTS',
|
||||
re: /\b(function|export|api surface|what does .* provide|helper|vv_\w+)\b/i },
|
||||
{ section: 'PURPOSE',
|
||||
re: /\b(what is|what does .* do|purpose|responsible for|job of)\b/i },
|
||||
];
|
||||
|
||||
const SECTION_BOOST = 0.06; // enough to reorder near-ties, not enough to beat a real match
|
||||
const KIND_BOOST = 0.02; // docs answer "how do I" better than a script header does
|
||||
|
||||
function detectIntent(q) {
|
||||
const hits = [];
|
||||
for (const i of INTENTS) if (i.re.test(q)) hits.push(i.section);
|
||||
return hits;
|
||||
}
|
||||
|
||||
function blobToVec(buf) {
|
||||
const b = Buffer.from(buf);
|
||||
return new Float32Array(b.buffer, b.byteOffset, b.length / 4);
|
||||
}
|
||||
|
||||
function dot(a, b) {
|
||||
let s = 0;
|
||||
for (let i = 0; i < a.length; i++) s += a[i] * b[i];
|
||||
return s;
|
||||
}
|
||||
|
||||
async function embedQuery(url, model, text, timeoutMs = 60000) {
|
||||
const res = await fetch(`${url.replace(/\/$/, '')}/api/embed`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ model, input: text }),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
if (!res.ok) throw new Error(`embed HTTP ${res.status}`);
|
||||
const j = await res.json();
|
||||
if (!j.embeddings || !j.embeddings[0]) throw new Error('embed returned no vector');
|
||||
return Float32Array.from(j.embeddings[0]);
|
||||
}
|
||||
|
||||
// Keep at most `perFile` chunks from any one file, so a single large document cannot fill the
|
||||
// entire context window and crowd out a better answer living somewhere else.
|
||||
function diversify(rows, k, perFile) {
|
||||
const seen = new Map();
|
||||
const out = [];
|
||||
for (const r of rows) {
|
||||
const n = seen.get(r.path) || 0;
|
||||
if (n >= perFile) continue;
|
||||
seen.set(r.path, n + 1);
|
||||
out.push(r);
|
||||
if (out.length >= k) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function search(opts) {
|
||||
const {
|
||||
dbPath, url, embedModel, query,
|
||||
k = 8, perFile = 3, section = null, kind = null, minScore = 0.0,
|
||||
} = opts;
|
||||
|
||||
const db = new DatabaseSync(dbPath, { readOnly: true });
|
||||
|
||||
let sql = 'SELECT id,path,kind,section,heading,part,content,vector FROM vv_chunks';
|
||||
const where = [], args = [];
|
||||
if (section) { where.push('section = ?'); args.push(section); }
|
||||
if (kind) { where.push('kind = ?'); args.push(kind); }
|
||||
if (where.length) sql += ' WHERE ' + where.join(' AND ');
|
||||
|
||||
const rows = db.prepare(sql).all(...args);
|
||||
if (!rows.length) { db.close(); return { results: [], intents: [], scanned: 0 }; }
|
||||
|
||||
const qv = await embedQuery(url, embedModel, query);
|
||||
let intents = section ? [] : detectIntent(query);
|
||||
|
||||
// "What is Varaverk" and "what is arr_sync.sh" are not the same question, and the PURPOSE
|
||||
// intent cannot tell them apart — it fires on both and boosts every PURPOSE block in the
|
||||
// repository at once. There are a couple of hundred, each genuinely describing the purpose of
|
||||
// something, and each a short sentence containing the word Varaverk. The project's own README
|
||||
// then loses to a script that migrates storage modes, because a paragraph is more diluted
|
||||
// than a one-line summary.
|
||||
//
|
||||
// A question that names the project and no component inside it is asking about the whole, so
|
||||
// PURPOSE is precisely the wrong section to promote. Dropping only that intent, rather than
|
||||
// all of them, leaves "why was Varaverk built this way" still routed to DESIGN PRINCIPLES.
|
||||
const namesProject = /\bvaraverk\b/i.test(query);
|
||||
const namesComponent = /\b[\w.-]+\.(sh|php|js)\b|\b[A-Z][A-Z0-9]*(_[A-Z0-9]+)+\b/.test(query);
|
||||
const projectLevel = namesProject && !namesComponent;
|
||||
if (projectLevel) intents = intents.filter(s => s !== 'PURPOSE');
|
||||
|
||||
// The same question wants the top-level prose, which is what the doc kinds are.
|
||||
const wantDoc = projectLevel
|
||||
|| /\b(how do i|steps|procedure|setup|install|troubleshoot|guide)\b/i.test(query);
|
||||
|
||||
const scored = rows.map(r => {
|
||||
let s = dot(qv, blobToVec(r.vector));
|
||||
if (intents.includes(r.section)) s += SECTION_BOOST;
|
||||
if (wantDoc && (r.kind === 'manual' || r.kind === 'readme')) s += KIND_BOOST;
|
||||
return {
|
||||
id: r.id, path: r.path, kind: r.kind, section: r.section,
|
||||
heading: r.heading, part: r.part, content: r.content, score: s,
|
||||
};
|
||||
});
|
||||
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
const kept = diversify(scored.filter(r => r.score >= minScore), k, perFile);
|
||||
db.close();
|
||||
return { results: kept, intents, scanned: rows.length };
|
||||
}
|
||||
|
||||
module.exports = { search, detectIntent, blobToVec, dot, embedQuery, INTENTS };
|
||||
Executable → Regular
+120
-8
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
|
||||
|
||||
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
|
||||
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
|
||||
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
|
||||
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
|
||||
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
|
||||
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
|
||||
touch "$CORRUPTION_SCAN_STRIKES_FILE"
|
||||
|
||||
@@ -301,6 +304,19 @@ reset_scan_strikes() {
|
||||
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
|
||||
}
|
||||
|
||||
# Bails out of the whole run without committing anything. Safe to call at any point before
|
||||
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
|
||||
# file exactly as the previous run left it and deletes nothing.
|
||||
abort_scan() {
|
||||
local why="$1"
|
||||
error "Corruption scan ABORTED — $why"
|
||||
error "No strikes recorded and nothing remediated this run — the library was not trusted."
|
||||
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
|
||||
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
|
||||
"Arr Corruption Scan" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
|
||||
# identical between Sonarr and Radarr.
|
||||
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
|
||||
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
||||
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
|
||||
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
||||
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
|
||||
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
|
||||
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
|
||||
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
|
||||
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
||||
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
||||
probe_file() {
|
||||
local host_path="$1" container_path output
|
||||
local host_path="$1" container_path output rc
|
||||
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
||||
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
||||
rc=$?
|
||||
|
||||
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
|
||||
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
|
||||
# header. A stopped container exits 1 with a daemon message; a missing binary exits
|
||||
# 127 — neither is evidence about the file, so both must be caught.
|
||||
if (( rc >= 125 )) \
|
||||
|| [[ "$output" == "Error response from daemon:"* \
|
||||
|| "$output" == "Cannot connect to the Docker daemon"* \
|
||||
|| "$output" == "error during connect:"* ]]; then
|
||||
echo "probe_error:${output//$'\n'/ }"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -z "$output" ]]; then
|
||||
echo "clean"
|
||||
else
|
||||
elif (( rc != 0 )); then
|
||||
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
|
||||
# contradictionary STSC and STCO. This is the only class that may be remediated.
|
||||
echo "corrupt:${output//$'\n'/ }"
|
||||
else
|
||||
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
|
||||
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
|
||||
# which says nothing about playability. Equating any stderr with corruption is what
|
||||
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
|
||||
# Reported for visibility, never strike-tracked, never remediated.
|
||||
echo "suspect:${output//$'\n'/ }"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
|
||||
TOTAL_CORRUPT=0
|
||||
TOTAL_REMEDIATED=0
|
||||
TOTAL_REMEDIATE_FAILED=0
|
||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||
|
||||
for arr in sonarr radarr; do
|
||||
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
||||
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
|
||||
STRIKE_HELD=0
|
||||
REMEDIATED=0
|
||||
REMEDIATE_FAILED=0
|
||||
PROBE_ERRORS=0
|
||||
SUSPECT_COUNT=0
|
||||
CONSECUTIVE_BAD=0
|
||||
QUEUE_PATH=()
|
||||
QUEUE_STRIKES=()
|
||||
QUEUE_ITEM=()
|
||||
|
||||
FRESH_CLEAN_TMP=$(mktemp)
|
||||
|
||||
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# A docker-level failure is not evidence about the file. Count it, never queue it.
|
||||
if [[ "$result" == probe_error:* ]]; then
|
||||
(( PROBE_ERRORS++ ))
|
||||
(( CONSECUTIVE_BAD++ ))
|
||||
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
|
||||
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$result" == "clean" ]]; then
|
||||
CONSECUTIVE_BAD=0
|
||||
reset_scan_strikes "$host_path"
|
||||
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
||||
continue
|
||||
fi
|
||||
|
||||
# corrupt:<reason>
|
||||
# A successful probe that merely warned. Proves the container is alive, so it clears
|
||||
# the consecutive-failure tripwire, but it never becomes a strike.
|
||||
if [[ "$result" == suspect:* ]]; then
|
||||
CONSECUTIVE_BAD=0
|
||||
(( SUSPECT_COUNT++ ))
|
||||
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
|
||||
continue
|
||||
fi
|
||||
|
||||
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
|
||||
# is deleted until this arr has been fully probed and the guards below have passed. A
|
||||
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
|
||||
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
|
||||
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
|
||||
reason="${result#corrupt:}"
|
||||
(( CORRUPT_COUNT++ ))
|
||||
strikes=$(increment_scan_strikes "$host_path")
|
||||
(( CONSECUTIVE_BAD++ ))
|
||||
|
||||
prev_strikes=$(get_scan_strikes "$host_path")
|
||||
prev_strikes="${prev_strikes//[^0-9]/}"
|
||||
strikes=$(( ${prev_strikes:-0} + 1 ))
|
||||
|
||||
QUEUE_PATH+=("$host_path")
|
||||
QUEUE_STRIKES+=("$strikes")
|
||||
QUEUE_ITEM+=("$item")
|
||||
|
||||
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
||||
|
||||
if [[ "$REMEDIATE" != true ]]; then
|
||||
continue
|
||||
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||
fi
|
||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||
|
||||
# ━━━ False-positive guards — run before anything is committed ━━━
|
||||
if (( CORRUPT_COUNT > 0 )); then
|
||||
# The pre-flight check only proves the container was up when the scan started.
|
||||
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
|
||||
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
|
||||
|
||||
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
|
||||
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
|
||||
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
|
||||
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
|
||||
for _q in "${!QUEUE_PATH[@]}"; do
|
||||
host_path="${QUEUE_PATH[$_q]}"
|
||||
strikes="${QUEUE_STRIKES[$_q]}"
|
||||
item="${QUEUE_ITEM[$_q]}"
|
||||
|
||||
set_scan_strikes "$host_path" "$strikes"
|
||||
|
||||
[[ "$REMEDIATE" != true ]] && continue
|
||||
|
||||
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
|
||||
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
|
||||
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
|
||||
fi
|
||||
reset_scan_strikes "$host_path"
|
||||
|
||||
file_id=$(echo "$item" | jq -r '.file_id')
|
||||
parent_id=$(echo "$item" | jq -r '.parent_id')
|
||||
title=$(echo "$item" | jq -r '.title')
|
||||
|
||||
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
||||
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
|
||||
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
||||
(( REMEDIATE_FAILED++ ))
|
||||
fi
|
||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||
done
|
||||
|
||||
merge_clean_state "$FRESH_CLEAN_TMP"
|
||||
rm -f "$FRESH_CLEAN_TMP"
|
||||
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
|
||||
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
|
||||
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
|
||||
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
|
||||
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
|
||||
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
|
||||
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
|
||||
ARR_REMEDIATED[$arr]=$REMEDIATED
|
||||
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
|
||||
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
|
||||
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
|
||||
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
|
||||
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
|
||||
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
|
||||
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
|
||||
if [[ "$REMEDIATE" == true ]]; then
|
||||
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
|
||||
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
|
||||
|
||||
@@ -146,10 +146,21 @@
|
||||
# (default: 7)
|
||||
#
|
||||
# DOWNLOAD_ORPHAN_MIN_VIDEO_MB
|
||||
# An entry with no video file above this size is JUNK (default: 50)
|
||||
# An entry with no video file above this size is JUNK (default: 50). Sonarr/Radarr only.
|
||||
#
|
||||
# DOWNLOAD_ORPHAN_MIN_AUDIO_MB
|
||||
# The same test for Lidarr (default: 2). Separate because a 50M floor would mark
|
||||
# every album folder as JUNK — single tracks rarely reach it.
|
||||
#
|
||||
# DOWNLOAD_ORPHAN_KEEP_MARKER
|
||||
# A file with this name inside a download folder pins it — the folder is never
|
||||
# classified or deleted (default: .vv-keep). For lossless rips the library holds
|
||||
# only at lower quality, which REDUNDANT would otherwise sweep.
|
||||
#
|
||||
# DOWNLOAD_ORPHAN_MAX_DELETE_GB
|
||||
# Abort the delete pass if the run total exceeds this (default: 100)
|
||||
# Per-run delete budget in GB (default: 100). A backlog above this is drained
|
||||
# safest-first (JUNK, then REDUNDANT, then UNMATCHED) up to the budget, and the
|
||||
# remainder is deferred to the next run rather than aborting the pass.
|
||||
#
|
||||
# SONARR_EXTENSIONS / RADARR_EXTENSIONS
|
||||
# Video extensions used to decide whether an entry contains real media
|
||||
@@ -161,7 +172,7 @@
|
||||
# arr_download_orphan_cleaner.sh — daily orchestrator entry
|
||||
# arr_download_orphan_cleaner.sh --dry-run — classify and report only
|
||||
# arr_download_orphan_cleaner.sh --status — show config and exit
|
||||
# arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB cap
|
||||
# arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB budget
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -197,6 +208,8 @@ fi
|
||||
|
||||
DOWNLOAD_ORPHAN_AGE="${DOWNLOAD_ORPHAN_AGE:-7}"
|
||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB="${DOWNLOAD_ORPHAN_MIN_VIDEO_MB:-50}"
|
||||
DOWNLOAD_ORPHAN_MIN_AUDIO_MB="${DOWNLOAD_ORPHAN_MIN_AUDIO_MB:-2}"
|
||||
DOWNLOAD_ORPHAN_KEEP_MARKER="${DOWNLOAD_ORPHAN_KEEP_MARKER:-.vv-keep}"
|
||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB="${DOWNLOAD_ORPHAN_MAX_DELETE_GB:-100}"
|
||||
|
||||
if [[ "${SHOW_STATUS:-false}" == true ]]; then
|
||||
@@ -204,9 +217,10 @@ if [[ "${SHOW_STATUS:-false}" == true ]]; then
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Enabled: ${DOWNLOAD_ORPHAN_CLEANER_ENABLED}"
|
||||
echo "$ICON_TIME Age gate: ${DOWNLOAD_ORPHAN_AGE}d"
|
||||
echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M"
|
||||
echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M video / ${DOWNLOAD_ORPHAN_MIN_AUDIO_MB}M audio"
|
||||
echo "$ICON_SHIELD Delete cap: ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G"
|
||||
for arr in SONARR RADARR; do
|
||||
echo "$ICON_SHIELD Keep marker: ${DOWNLOAD_ORPHAN_KEEP_MARKER}"
|
||||
for arr in SONARR RADARR LIDARR; do
|
||||
dir_var="${MY_ID}_${arr}_DOWNLOAD_DIR"
|
||||
echo "$ICON_CLEAN ${arr}: ${!dir_var:-<not configured>}"
|
||||
done
|
||||
@@ -221,13 +235,16 @@ AGE_CUTOFF=$(( $(date +%s) - DOWNLOAD_ORPHAN_AGE * 86400 ))
|
||||
TOTAL_DELETED=0
|
||||
TOTAL_DELETED_MB=0
|
||||
TOTAL_HELD=0
|
||||
TOTAL_DEFERRED=0
|
||||
TOTAL_KEPT=0
|
||||
TOTAL_SCANS=0
|
||||
|
||||
echo "━━━━━ $ICON_CLEAN DOWNLOAD ORPHAN CLEANER ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
[[ "$DRY_RUN" == true ]] && echo "$ICON_SKIP DRY RUN — nothing will be deleted or imported"
|
||||
|
||||
for arr in sonarr radarr; do
|
||||
for arr in sonarr radarr lidarr; do
|
||||
api_ver="v3"; [[ "$arr" == "lidarr" ]] && api_ver="v1"
|
||||
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
||||
arr_url="${!url_var:-}"; arr_key="${!key_var:-}"
|
||||
dir_var="${MY_ID}_${arr^^}_DOWNLOAD_DIR"
|
||||
@@ -247,24 +264,38 @@ for arr in sonarr radarr; do
|
||||
echo "━━━ $ICON_SYNC ${arr^} — $dl_dir ━━━"
|
||||
|
||||
ver_var="${arr^^}_VERSION_MAJOR"
|
||||
check_arr_version "$arr_url" "$arr_key" "v3" "${!ver_var}" "${arr^}" || {
|
||||
check_arr_version "$arr_url" "$arr_key" "$api_ver" "${!ver_var}" "${arr^}" || {
|
||||
warn "${arr^} version check failed — skipping this arr"
|
||||
continue
|
||||
}
|
||||
# min_mb is per-arr because the JUNK test is "contains no real media file". A 50MB floor
|
||||
# is right for video and catastrophic for audio — most single tracks never reach it, so
|
||||
# every music folder would classify as JUNK and be deleted regardless of import state.
|
||||
case "$arr" in
|
||||
sonarr)
|
||||
queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true"
|
||||
exts_var="SONARR_EXTENSIONS"
|
||||
scan_command="DownloadedEpisodesScan"
|
||||
library_endpoint="series"
|
||||
min_mb="$DOWNLOAD_ORPHAN_MIN_VIDEO_MB"
|
||||
;;
|
||||
radarr)
|
||||
queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true"
|
||||
exts_var="RADARR_EXTENSIONS"
|
||||
scan_command="DownloadedMoviesScan"
|
||||
library_endpoint="movie"
|
||||
min_mb="$DOWNLOAD_ORPHAN_MIN_VIDEO_MB"
|
||||
;;
|
||||
lidarr)
|
||||
queue_endpoint="queue?pageSize=1000&includeUnknownArtistItems=true"
|
||||
exts_var="LIDARR_EXTENSIONS"
|
||||
scan_command="DownloadedAlbumsScan"
|
||||
library_endpoint="artist"
|
||||
min_mb="$DOWNLOAD_ORPHAN_MIN_AUDIO_MB"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$arr" == "sonarr" ]]; then
|
||||
queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true"
|
||||
exts_var="SONARR_EXTENSIONS"
|
||||
scan_command="DownloadedEpisodesScan"
|
||||
library_endpoint="series"
|
||||
else
|
||||
queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true"
|
||||
exts_var="RADARR_EXTENSIONS"
|
||||
scan_command="DownloadedMoviesScan"
|
||||
library_endpoint="movie"
|
||||
fi
|
||||
|
||||
QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "v3" "$queue_endpoint" "${arr^}") || {
|
||||
QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "$api_ver" "$queue_endpoint" "${arr^}") || {
|
||||
error "${arr^} queue fetch failed — cannot tell tracked from orphaned, skipping this arr"
|
||||
continue
|
||||
}
|
||||
@@ -282,11 +313,21 @@ for arr in sonarr radarr; do
|
||||
SCAN_PATHS=()
|
||||
UNMATCHED_PATHS=()
|
||||
UNMATCHED_SIZES=()
|
||||
arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0
|
||||
arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0; arr_kept=0
|
||||
|
||||
while IFS= read -r entry; do
|
||||
base="${entry##*/}"
|
||||
|
||||
# An operator keep-marker outranks every verdict below. Needed because REDUNDANT only
|
||||
# asks "does the library hold this album", not "at what quality" — a lossless rip whose
|
||||
# library copy is MP3 is redundant by that test and would be swept on the next run.
|
||||
# The marker is a file inside the folder rather than a conf list so it survives renames
|
||||
# and cannot drift out of sync with what is actually on disk.
|
||||
if [[ -e "$entry/$DOWNLOAD_ORPHAN_KEEP_MARKER" ]]; then
|
||||
arr_kept=$((arr_kept + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ -n "${PROTECTED[$base]:-}" ]]; then
|
||||
arr_tracked=$((arr_tracked + 1))
|
||||
continue
|
||||
@@ -298,18 +339,30 @@ for arr in sonarr radarr; do
|
||||
continue
|
||||
fi
|
||||
|
||||
has_video=false
|
||||
# JUNK means "holds no real media". That verdict is only as good as the extension
|
||||
# list, and a missing extension turns real content into a delete — 2026-08-21 the
|
||||
# audio list had no "wv", which classified 23 folders of WavPack lossless (1.5G per
|
||||
# file) as junk. So a folder with large files that are merely *unrecognised* is held
|
||||
# for review, never deleted; only a folder with nothing big in it at all is junk.
|
||||
has_media=false
|
||||
big_unknown=0
|
||||
while IFS= read -r f; do
|
||||
if has_extension "$f" "${arr_exts[@]}"; then
|
||||
has_video=true
|
||||
has_media=true
|
||||
break
|
||||
fi
|
||||
done < <(find "$entry" -type f -size +"${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}"M 2>/dev/null)
|
||||
big_unknown=$((big_unknown + 1))
|
||||
done < <(find "$entry" -type f -size +"${min_mb}"M 2>/dev/null)
|
||||
|
||||
size_mb=$(du -sm "$entry" 2>/dev/null | cut -f1)
|
||||
size_mb=${size_mb:-0}
|
||||
size_mb=$(dir_size_mb "$entry") || size_mb=0
|
||||
|
||||
if [[ "$has_video" == false ]]; then
|
||||
if [[ "$has_media" == false ]] && (( big_unknown > 0 )); then
|
||||
warn " no recognised media, but $big_unknown large file(s) of unknown type — holding: $base"
|
||||
arr_held=$((arr_held + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$has_media" == false ]]; then
|
||||
DELETE_PATHS+=("$entry")
|
||||
DELETE_SIZES+=("$size_mb")
|
||||
DELETE_LABELS+=("JUNK")
|
||||
@@ -318,20 +371,43 @@ for arr in sonarr radarr; do
|
||||
fi
|
||||
|
||||
enc_title=$(jq -rn --arg t "$base" '$t|@uri')
|
||||
parse=$(arr_api "$arr_url" "$arr_key" "v3" "parse?title=${enc_title}" "${arr^}") || {
|
||||
parse=$(arr_api "$arr_url" "$arr_key" "$api_ver" "parse?title=${enc_title}" "${arr^}") || {
|
||||
warn " parse failed for: $base — holding"
|
||||
arr_held=$((arr_held + 1))
|
||||
continue
|
||||
}
|
||||
|
||||
if [[ "$arr" == "sonarr" ]]; then
|
||||
matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)')
|
||||
missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length')
|
||||
else
|
||||
# Radarr's parse never populates hasFile — movieFileId is the reliable signal
|
||||
matched=$(echo "$parse" | jq '.movie != null')
|
||||
missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end')
|
||||
fi
|
||||
case "$arr" in
|
||||
sonarr)
|
||||
matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)')
|
||||
missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length')
|
||||
;;
|
||||
radarr)
|
||||
# Radarr's parse never populates hasFile — movieFileId is the reliable signal
|
||||
matched=$(echo "$parse" | jq '.movie != null')
|
||||
missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end')
|
||||
;;
|
||||
lidarr)
|
||||
# Lidarr's parse returns albums with statistics:null, so the track count has
|
||||
# to be read back from album/{id} — the same shape of gap as Radarr's hasFile.
|
||||
matched=$(echo "$parse" | jq '(.artist != null) and ((.albums | length) > 0)')
|
||||
missing=1
|
||||
if [[ "$matched" == true ]]; then
|
||||
album_id=$(echo "$parse" | jq -r '.albums[0].id // empty')
|
||||
if [[ -z "$album_id" ]]; then
|
||||
warn " parse matched but returned no album id: $base — holding"
|
||||
arr_held=$((arr_held + 1))
|
||||
continue
|
||||
fi
|
||||
album_json=$(arr_api "$arr_url" "$arr_key" "$api_ver" "album/$album_id" "${arr^}") || {
|
||||
warn " album lookup failed for: $base — holding"
|
||||
arr_held=$((arr_held + 1))
|
||||
continue
|
||||
}
|
||||
missing=$(echo "$album_json" | jq 'if ((.statistics.trackFileCount // 0) > 0) then 0 else 1 end')
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$matched" != true ]]; then
|
||||
UNMATCHED_PATHS+=("$entry")
|
||||
@@ -354,7 +430,7 @@ for arr in sonarr radarr; do
|
||||
# fails on a small batch that is legitimately all-unmatched, which is the normal case
|
||||
# once daily runs have caught up.
|
||||
if (( ${#UNMATCHED_PATHS[@]} > 0 )); then
|
||||
library_count=$(arr_api "$arr_url" "$arr_key" "v3" "$library_endpoint" "${arr^}" | jq 'length' 2>/dev/null)
|
||||
library_count=$(arr_api "$arr_url" "$arr_key" "$api_ver" "$library_endpoint" "${arr^}" | jq 'length' 2>/dev/null)
|
||||
if [[ ! "$library_count" =~ ^[0-9]+$ ]] || (( library_count == 0 )); then
|
||||
warn " ${arr^}: library reports ${library_count:-no} titles — cannot trust 'no match', holding ${#UNMATCHED_PATHS[@]} unmatched"
|
||||
arr_held=$((arr_held + ${#UNMATCHED_PATHS[@]}))
|
||||
@@ -368,26 +444,59 @@ for arr in sonarr radarr; do
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( arr_delete_mb / 1024 > DOWNLOAD_ORPHAN_MAX_DELETE_GB )) && [[ "$I_KNOW" != true ]]; then
|
||||
error "${arr^}: delete total $((arr_delete_mb / 1024))G exceeds cap of ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G — aborting delete pass"
|
||||
notify "${arr^} download orphan delete total $((arr_delete_mb / 1024))G exceeds ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap on $(hostname) — possible partial queue data, nothing deleted. Re-run with --i-know-what-im-doing if legitimate." \
|
||||
# The cap is a per-run risk budget, not a reason to do nothing. Aborting the whole pass
|
||||
# once the backlog exceeds it is self-defeating: the backlog can never shrink below the
|
||||
# cap on its own, so every later run aborts too and the pool fills anyway (exactly how
|
||||
# 347G accumulated here by 2026-08-21). Delete in ascending order of risk instead, stop
|
||||
# at the cap, and defer the rest to the next run so a backlog drains over days.
|
||||
#
|
||||
# Live downloads are already protected by DOWNLOAD_ORPHAN_AGE, not by this cap — anything
|
||||
# in flight is younger than the age gate and never reaches classification. That is what
|
||||
# makes draining safe: the partial-queue-data case the cap was written for cannot put a
|
||||
# still-downloading entry in these arrays.
|
||||
cap_mb=$((DOWNLOAD_ORPHAN_MAX_DELETE_GB * 1024))
|
||||
cap_active=true
|
||||
[[ "$I_KNOW" == true || "$DRY_RUN" == true ]] && cap_active=false
|
||||
|
||||
arr_deferred=0; arr_deferred_mb=0; arr_run_mb=0
|
||||
|
||||
if [[ "$cap_active" == true ]] && (( arr_delete_mb > cap_mb )); then
|
||||
warn " ${arr^}: $((arr_delete_mb / 1024))G classified vs ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap — deleting safest-first up to the cap, deferring the rest"
|
||||
notify "${arr^} download orphan backlog is $((arr_delete_mb / 1024))G on $(hostname), above the ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G per-run cap. Draining safest-first; the remainder follows on later runs. Re-run with --i-know-what-im-doing to clear it in one pass." \
|
||||
"Download Orphan Cleaner" "warning"
|
||||
unset PROTECTED
|
||||
continue
|
||||
fi
|
||||
|
||||
for i in "${!DELETE_PATHS[@]}"; do
|
||||
entry="${DELETE_PATHS[$i]}"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
||||
else
|
||||
echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
||||
rm -rf "$entry"
|
||||
fi
|
||||
TOTAL_DELETED=$((TOTAL_DELETED + 1))
|
||||
TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i]))
|
||||
# JUNK first (no media at all), then REDUNDANT (parse-verified already in the library),
|
||||
# then UNMATCHED last — it rests on "the arr does not know this title", the weakest of
|
||||
# the three signals, so it is the first thing the cap defers.
|
||||
for pass in JUNK REDUNDANT UNMATCHED; do
|
||||
for i in "${!DELETE_PATHS[@]}"; do
|
||||
[[ "${DELETE_LABELS[$i]}" == "$pass" ]] || continue
|
||||
entry="${DELETE_PATHS[$i]}"
|
||||
|
||||
if [[ "$cap_active" == true ]] && (( arr_run_mb + DELETE_SIZES[i] > cap_mb )); then
|
||||
arr_deferred=$((arr_deferred + 1))
|
||||
arr_deferred_mb=$((arr_deferred_mb + DELETE_SIZES[i]))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
||||
else
|
||||
echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
||||
rm -rf "$entry"
|
||||
fi
|
||||
arr_run_mb=$((arr_run_mb + DELETE_SIZES[i]))
|
||||
TOTAL_DELETED=$((TOTAL_DELETED + 1))
|
||||
TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i]))
|
||||
done
|
||||
done
|
||||
|
||||
if (( arr_deferred > 0 )); then
|
||||
echo " $ICON_WARN ${arr^}: deferred $arr_deferred entries ($((arr_deferred_mb / 1024))G) to the next run — cap reached"
|
||||
TOTAL_DEFERRED=$((TOTAL_DEFERRED + arr_deferred))
|
||||
fi
|
||||
|
||||
for base in "${SCAN_PATHS[@]}"; do
|
||||
if [[ -z "$container_dir" ]]; then
|
||||
echo " $ICON_WARN IMPORTABLE but ${cdir_var} not set — holding: $base"
|
||||
@@ -406,8 +515,9 @@ for arr in sonarr radarr; do
|
||||
fi
|
||||
done
|
||||
|
||||
echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_recent recent, ${#DELETE_PATHS[@]} deleted ($((arr_delete_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held"
|
||||
echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_kept kept, $arr_recent recent, $((${#DELETE_PATHS[@]} - arr_deferred)) deleted ($((arr_run_mb / 1024))G), $arr_deferred deferred ($((arr_deferred_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held"
|
||||
TOTAL_HELD=$((TOTAL_HELD + arr_held))
|
||||
TOTAL_KEPT=$((TOTAL_KEPT + arr_kept))
|
||||
unset PROTECTED
|
||||
done
|
||||
|
||||
@@ -416,6 +526,8 @@ echo "━━━━━ $ICON_DONE SUMMARY ━━━━━"
|
||||
echo "$ICON_TRASH Deleted: $TOTAL_DELETED ($((TOTAL_DELETED_MB / 1024))G)"
|
||||
echo "$ICON_RUN Import scans: $TOTAL_SCANS"
|
||||
echo "$ICON_WARN Held: $TOTAL_HELD"
|
||||
echo "$ICON_SKIP Deferred: $TOTAL_DEFERRED"
|
||||
echo "$ICON_SHIELD Kept (marker): $TOTAL_KEPT"
|
||||
|
||||
# Held alone never notifies — there is always something awaiting review, and on a daily
|
||||
# schedule that would be a notification every morning saying nothing happened.
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Remote API Access — Cache-First, SSH Fallback
|
||||
# If conf_sync.sh has populated /tmp/.cache/vv/d/ and
|
||||
# If conf_sync.sh has populated /tmp/varaverk/conf/ and
|
||||
# load_config.sh has sourced it, HOST*_<ARR>_API_KEY vars are available
|
||||
# in the environment. Remote functions use them to call the arr API
|
||||
# directly over Tailscale (no SSH, no remote shell). If the cached key
|
||||
|
||||
Executable → Regular
+71
-9
@@ -327,7 +327,7 @@ ARTIST_RESPONSE=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY"
|
||||
}
|
||||
|
||||
ARTIST_IDS=$(echo "$ARTIST_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
ARTIST_COUNT=$(echo "$ARTIST_IDS" | grep -c "[0-9]" 2>/dev/null || echo 0)
|
||||
ARTIST_COUNT=$(echo "$ARTIST_IDS" | grep -c "[0-9]" 2>/dev/null || true)
|
||||
|
||||
# Safety Layer 4 — artist count > 0
|
||||
if [[ "$ARTIST_COUNT" -eq 0 ]]; then
|
||||
@@ -457,6 +457,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
|
||||
"Lidarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Lidarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
|
||||
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
fi
|
||||
|
||||
@@ -541,7 +541,7 @@ echo "━━━ $ICON_SYNC Existing Libraries ━━━"
|
||||
# fetch when it's stale, and waits out an active rescan before either.
|
||||
LIDARR_ARTISTS_JSON=$(arr_get_tracked_data "lidarr" "$LIDARR_URL" "$LIDARR_API_KEY" "v1") || { error "Could not fetch Lidarr artists"; exit 1; }
|
||||
LIDARR_NAMES=$(echo "$LIDARR_ARTISTS_JSON" | jq -r '.[].artistName' 2>/dev/null)
|
||||
LIDARR_COUNT=$(echo "$LIDARR_NAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
LIDARR_COUNT=$(echo "$LIDARR_NAMES" | grep -c . 2>/dev/null || true)
|
||||
log "$LIDARR_COUNT artists in Lidarr"
|
||||
|
||||
EMBY_LIBRARY_JSON=$(emby_api "Items?IncludeItemTypes=MusicAlbum&Recursive=true&Fields=AlbumArtists&Limit=10000") || {
|
||||
@@ -549,7 +549,7 @@ EMBY_LIBRARY_JSON=$(emby_api "Items?IncludeItemTypes=MusicAlbum&Recursive=true&F
|
||||
EMBY_ARTIST_NAMES=""
|
||||
}
|
||||
EMBY_ARTIST_NAMES=$(echo "$EMBY_LIBRARY_JSON" | jq -r '.Items[] | .AlbumArtists[]?.Name' 2>/dev/null)
|
||||
EMBY_ARTIST_COUNT=$(echo "$EMBY_ARTIST_NAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
EMBY_ARTIST_COUNT=$(echo "$EMBY_ARTIST_NAMES" | grep -c . 2>/dev/null || true)
|
||||
log "$EMBY_ARTIST_COUNT album artists in Emby library"
|
||||
|
||||
# MusicBrainz's canonical name for some artists (e.g. "blink‐182") uses a Unicode
|
||||
|
||||
Executable → Regular
+135
-9
@@ -314,7 +314,7 @@ MOVIES_RESPONSE=$(arr_get_tracked_data "radarr" "$RADARR_URL" "$RADARR_API_KEY"
|
||||
}
|
||||
|
||||
MOVIE_IDS=$(echo "$MOVIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || echo 0)
|
||||
MOVIE_COUNT=$(echo "$MOVIE_IDS" | grep -c "." 2>/dev/null || true)
|
||||
|
||||
# Safety Layer 4 — movie count > 0
|
||||
if [[ "$MOVIE_COUNT" -eq 0 ]]; then
|
||||
@@ -450,9 +450,46 @@ NOW=$(date +%s)
|
||||
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
|
||||
# second time (2026-07-17) — the size-threshold check below needs to know the total before
|
||||
# deleting anything, not before knowing what to delete.
|
||||
# Carries size and ctime alongside the path now, because the budget pass below has to order by
|
||||
# age and stop at a byte ceiling — neither of which a bare path list can answer.
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
|
||||
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
|
||||
# partial failure underneath that threshold — one root folder failing to enumerate makes its
|
||||
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
|
||||
# not reproduce on the next run.
|
||||
#
|
||||
# The file is REBUILT from this run's classifications rather than edited in place, which is what
|
||||
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
|
||||
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
|
||||
#
|
||||
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
|
||||
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
|
||||
# so every write truncated the store to one line. See common.sh.
|
||||
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
# Records this run's strike for a file and says whether it has served enough of them.
|
||||
# Returns 0 when the file may be deleted, 1 when it is still accruing.
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -504,13 +543,94 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
|
||||
# is one — but it is not going to be deleted this run, so it must not appear in the denominator
|
||||
# the budget reports against or the run claims to have skipped work it never queued.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
|
||||
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
|
||||
# advanced real strike counters would make the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
|
||||
# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
|
||||
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
|
||||
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
|
||||
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
|
||||
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
|
||||
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
|
||||
# the run removes exactly the same files — the log just loses a paragraph.
|
||||
#
|
||||
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
|
||||
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
|
||||
# is what a partnership merge against a partner holding older copies produces. That distinction
|
||||
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
|
||||
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
|
||||
_ai_ev=$(awk -F'\t' '
|
||||
{ n++; bytes += $1
|
||||
c = int($2)
|
||||
if (cmin == 0 || c < cmin) cmin = c
|
||||
if (c > cmax) cmax = c
|
||||
bucket[int(c / 21600)]++ }
|
||||
END {
|
||||
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
|
||||
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
|
||||
n, bytes/1073741824, (cmax-cmin)/3600, top
|
||||
}' "$TO_DELETE_FILE")
|
||||
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
|
||||
| while IFS= read -r p; do [[ -f "$p" ]] && \
|
||||
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
|
||||
|
||||
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
|
||||
|
||||
EVIDENCE
|
||||
$_ai_ev
|
||||
sample (modification month, then path):
|
||||
$_ai_mt
|
||||
|
||||
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
|
||||
|
||||
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
|
||||
|
||||
if [[ -n "$_ai_note" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
|
||||
printf '%s\n' "$_ai_note"
|
||||
fi
|
||||
unset _ai_ev _ai_mt
|
||||
fi
|
||||
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
|
||||
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
# One file larger than the whole budget can never fit, so it would be re-found and
|
||||
# re-deferred every night. Name it rather than loop on it silently.
|
||||
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
|
||||
"Radarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
|
||||
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
|
||||
"Radarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With a budget in force those differ, and
|
||||
# reporting the classification as the outcome is the oldest bug shape in this codebase.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
|
||||
"Radarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
@@ -319,6 +319,31 @@ if [[ "$ENABLE_LOGGING" == true ]]; then
|
||||
" [\(if .forward_anime_miss then "FORWARD-ANIME" elif .forward_kids_miss then "FORWARD-KIDS" elif .reverse_anime_leak then "REVERSE-ANIME" elif .reverse_kids_leak then "REVERSE-KIDS" else "?" end)] \(.title) (root: \(.rootFolderPath), network: \(.network // "n/a"), cert: \(.certification // "n/a"))"'
|
||||
fi
|
||||
|
||||
# ── The reverse leaks, written down ───────────────────────────────────────────────────────────
|
||||
# This block's own header says it does not act on REVERSE-ANIME leaks because they are genuine
|
||||
# judgement calls. That is right, and it is also why they are the one result worth persisting:
|
||||
# every other bucket either self-resolves or is acted on by --move, while these accumulate as a
|
||||
# number in a summary nobody can do anything with. Seventeen of them hid two live-action dramas
|
||||
# filed under anime for as long as the count stayed a count.
|
||||
#
|
||||
# Written as the script's own verdict so anything reading it — the triage that reads this next —
|
||||
# inherits the classification rather than computing a second opinion from the same metadata.
|
||||
# Report-only: this records what was found, it does not change what happens to any of it.
|
||||
if [[ -n "${STATE_DIR:-}" ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
_review_file="$STATE_DIR/arr_classification_review.json"
|
||||
echo "$RESULTS" | jq -c --arg host "$MY_ID" --argjson ts "$(date +%s)" '
|
||||
{ host: $host, ts: $ts, arr: "sonarr",
|
||||
reverse_anime: [ .[] | select(.reverse_anime_leak) |
|
||||
{ title, root: .rootFolderPath, network: (.network // ""), cert: (.certification // ""),
|
||||
lang: (.originalLanguage.name // .originalLanguage // ""), id: .id } ],
|
||||
reverse_kids: [ .[] | select(.reverse_kids_leak) |
|
||||
{ title, root: .rootFolderPath, network: (.network // ""), cert: (.certification // "") } ] }
|
||||
' > "$_review_file" 2>/dev/null \
|
||||
&& log "$ICON_GEAR Review list written — $REVERSE_ANIME_COUNT anime leak(s) for triage" \
|
||||
|| warn "Could not write $_review_file — triage will have nothing to read"
|
||||
unset _review_file
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
Executable → Regular
+71
-9
@@ -308,7 +308,7 @@ SERIES_RESPONSE=$(arr_get_tracked_data "sonarr" "$SONARR_URL" "$SONARR_API_KEY"
|
||||
}
|
||||
|
||||
SERIES_IDS=$(echo "$SERIES_RESPONSE" | jq -r '.[].id' 2>/dev/null)
|
||||
SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || echo 0)
|
||||
SERIES_COUNT=$(echo "$SERIES_IDS" | grep -c "." 2>/dev/null || true)
|
||||
|
||||
# Safety Layer 4 — series count > 0
|
||||
if [[ "$SERIES_COUNT" -eq 0 ]]; then
|
||||
@@ -440,6 +440,30 @@ NOW=$(date +%s)
|
||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||
> "$TO_DELETE_FILE"
|
||||
|
||||
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||
# rebuilt from each run rather than edited, which is what prunes it.
|
||||
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
|
||||
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||
> "$STRIKES_NEW"
|
||||
HELD_COUNT=0
|
||||
HELD_BYTES=0
|
||||
|
||||
orphan_strike_ok() {
|
||||
local path="$1" prev strikes
|
||||
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||
strikes=$(( ${prev:-0} + 1 ))
|
||||
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||
return 1
|
||||
}
|
||||
|
||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||
(( ORPHAN_COUNT++ ))
|
||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
else
|
||||
log "JUNK: $filepath"
|
||||
(( JUNK_COUNT++ ))
|
||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
||||
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||
fi
|
||||
|
||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||
@@ -491,13 +517,43 @@ done < <(
|
||||
done | sort -u
|
||||
)
|
||||
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||
# run, so it must not appear in the denominator the budget reports against.
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||
|
||||
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||
# the next real run delete a run early.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||
# ==============================================================================================
|
||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
|
||||
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||
|
||||
if [[ "$I_KNOW" == true ]]; then
|
||||
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||
else
|
||||
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
|
||||
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||
error " $_BUDGET_STUCK"
|
||||
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||
"Sonarr Cleanup" "normal"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
while IFS= read -r filepath; do
|
||||
[[ -z "$filepath" ]] && continue
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
done < "$TO_DELETE_FILE"
|
||||
done < "$BUDGET_FILE"
|
||||
|
||||
info "Cleaning up empty folders..."
|
||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Clean — nothing to remove"
|
||||
else
|
||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||
"Sonarr Cleanup" "warning"
|
||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||
notify_emby_scan
|
||||
|
||||
@@ -135,6 +135,28 @@ and a third node would work with no template change at all.
|
||||
|--------|------|-------------|
|
||||
| `conf_upgrade.sh` | Merge template into conf — structure forward, values preserved | Automatically, after every `git pull` |
|
||||
| `conf_populate.sh` | Detect settings from running services into the host conf | Manually — onboarding, or after a key rotation |
|
||||
| `migrate_data_layout.sh` | Move everything persisted into the rooted `data/` tree | Once per host, manually. Idempotent. |
|
||||
|
||||
### 📦 Data layout migration — `migrate_data_layout.sh`
|
||||
|
||||
`conf_upgrade.sh` adds keys the template has and the installation does not; it never rewrites a
|
||||
value you already have. That is exactly what you want from it, and exactly why it cannot perform
|
||||
a layout migration — the paths being moved are *existing* keys, so their values would keep
|
||||
pointing at the old layout forever while the new directory variables sat beside them unused.
|
||||
|
||||
So this rewrites those values and moves the files to match. Both halves or neither.
|
||||
|
||||
```bash
|
||||
Deployment/migrate_data_layout.sh --dry-run # always first
|
||||
Deployment/migrate_data_layout.sh
|
||||
```
|
||||
|
||||
It refuses to run while a job from *this* installation is active — scoped to the installation's
|
||||
own path, because `pgrep` is system-wide and a box running both a production checkout and a
|
||||
development clone will otherwise always look busy. `--force` overrides.
|
||||
|
||||
Each host runs it itself: `data/` is gitignored, so a restructure travels as code and conf while
|
||||
the files stay where they are. See `data/README.md` for the resulting layout.
|
||||
|
||||
| Template | Role |
|
||||
|----------|------|
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
# EMPTY fields, never overwrites existing values unless --overwrite is passed.
|
||||
#
|
||||
# After populating, pushes the updated conf to all partners via conf_sync.sh
|
||||
# so they have the fresh keys in their /tmp/.cache/vv/d/ cache immediately.
|
||||
# so they have the fresh keys in their /tmp/varaverk/conf/ cache immediately.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# AUTO-DETECTED FIELDS
|
||||
@@ -158,7 +158,11 @@
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
# Deployment/ sits one level under the repo root, not three. The old ../../../ resolved to
|
||||
# /boot/config on a flash install and /mnt/user on an appdata one — outside the repo either
|
||||
# way, so this sourced nothing and every helper below was "command not found". Onboard Step 11
|
||||
# has been failing on that since it was written.
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
SCRIPTS_ROOT="$SCRIPTS_DIR"
|
||||
|
||||
OVERWRITE=false
|
||||
|
||||
+179
-29
@@ -69,6 +69,37 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# The Host Slot Must Be Declared, Never Inferred
|
||||
# host.conf.template ships HOSTN_ placeholders; a live host conf uses HOST1_ / HOST2_. Key
|
||||
# matching is literal, so merging the raw template against a real host conf classifies every
|
||||
# existing key as deprecated — KEPT 0 — and the install drops every credential in the file.
|
||||
# Verified on HOST1 2026-08-02: it would have removed HOST1_RADARR_API_KEY,
|
||||
# HOST1_EMBY_API_KEY, HOST1_NPM_PASS and 137 others.
|
||||
#
|
||||
# --host-slot HOST<n> is how a caller says which slot the template is being resolved for, and
|
||||
# it works for any slot — HOST1, HOST2, and whatever a third server would be. The slot is
|
||||
# still never inferred from the target and applied silently: the caller declares it, and the
|
||||
# target is read only to contradict a wrong answer. A declared slot that disagrees with the
|
||||
# target's own keys aborts, because substituting for the wrong slot destroys the file just as
|
||||
# thoroughly as not substituting at all. Without the flag, a template containing HOSTN is
|
||||
# refused exactly as before.
|
||||
#
|
||||
# Both cases are substituted. HOSTN_ covers the key prefixes, bare HOSTN appears in section
|
||||
# comments, and lowercase hostn is a real value — the hostn-appdata rsync profile keys. A
|
||||
# substitution handling only HOSTN_ leaves a conf carrying a profile named hostn-appdata that
|
||||
# nothing references.
|
||||
#
|
||||
# A Target Owning Two Slots Is Refused
|
||||
# A host conf describes exactly one server. Finding both HOST1_ and HOST2_ key definitions in
|
||||
# one target means it is not the file it claims to be, so the slot cross-check has nothing
|
||||
# trustworthy to compare against and the run aborts rather than picking one.
|
||||
#
|
||||
# Total Mismatch Is Refused
|
||||
# Keeping nothing from a populated conf is never a real upgrade; it means the two files do
|
||||
# not describe the same thing. KEPT 0 with a non-empty REMOVED list aborts. This is the
|
||||
# general net behind the HOSTN check — the failure mode is silent and total, so it fires in
|
||||
# --dry-run as well, putting the warning in the report itself.
|
||||
#
|
||||
# Root Required to Write
|
||||
# Installing over the target requires root. --dry-run deliberately does not, so the
|
||||
# change report can be previewed by anyone.
|
||||
@@ -104,10 +135,13 @@
|
||||
#
|
||||
# No conf vars. All inputs are CLI flags.
|
||||
#
|
||||
# --template <file> New version conf file (source of structure and defaults)
|
||||
# --target <file> Existing user conf (source of real values — always preserved)
|
||||
# --dry-run Show what would change without writing
|
||||
# --backup Write a .bak copy of target before modifying
|
||||
# --template <file> New version conf file (source of structure and defaults)
|
||||
# --target <file> Existing user conf (source of real values — always preserved)
|
||||
# --host-slot <HOSTn> Resolve HOSTN/hostn placeholders to this slot before merging.
|
||||
# Required for host.conf.template; meaningless for master.conf.template,
|
||||
# which has no placeholders. Must match the target's own slot.
|
||||
# --dry-run Show what would change without writing
|
||||
# --backup Write a .bak copy of target before modifying
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
@@ -122,6 +156,11 @@
|
||||
# conf_upgrade.sh --template Deployment/master.conf.template --target Configurations/master.conf
|
||||
# Apply the upgrade in-place with no backup.
|
||||
#
|
||||
# conf_upgrade.sh --template Deployment/host.conf.template --target Configurations/host1.conf \
|
||||
# --host-slot HOST1 --dry-run
|
||||
# Preview a host conf upgrade. HOSTN/hostn are resolved to HOST1/host1 first. Swap in HOST2
|
||||
# and host2.conf for the other server — the template is the same file for every slot.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
@@ -132,13 +171,16 @@ TEMPLATE=""
|
||||
TARGET=""
|
||||
DRY_RUN=false
|
||||
BACKUP=false
|
||||
HOST_SLOT=""
|
||||
_RESOLVED_TMPL="" # set only when --host-slot triggers a substitution; cleaned on exit
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--template) TEMPLATE="$2"; shift 2 ;;
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
--backup) BACKUP=true; shift ;;
|
||||
--template) TEMPLATE="$2"; shift 2 ;;
|
||||
--target) TARGET="$2"; shift 2 ;;
|
||||
--host-slot) HOST_SLOT="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
--backup) BACKUP=true; shift ;;
|
||||
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
@@ -148,6 +190,80 @@ done
|
||||
[[ -f "$TEMPLATE" ]] || { echo "Error: template not found: $TEMPLATE" >&2; exit 1; }
|
||||
[[ -f "$TARGET" ]] || { echo "Error: target not found: $TARGET" >&2; exit 1; }
|
||||
|
||||
# ── Host slot resolution ─────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# host.conf.template ships HOSTN_ placeholders; a live host conf uses HOST1_ / HOST2_. Key
|
||||
# matching below is literal, so merging the raw template against a real host conf classifies
|
||||
# EVERY existing key as deprecated and every template key as new — KEPT 0, and the install
|
||||
# would drop every credential in the file.
|
||||
#
|
||||
# --host-slot is how a caller declares which slot the template is for. The slot is never
|
||||
# inferred from the target and silently applied: the caller states it, and the target is used
|
||||
# only to contradict a wrong answer. Substituting for the wrong slot is the same catastrophe as
|
||||
# not substituting at all, so a declared slot that disagrees with the target is refused.
|
||||
#
|
||||
# Both cases matter. HOSTN_ covers the 159 key prefixes; bare HOSTN appears in section comments,
|
||||
# and lowercase hostn is a real value — the hostn-appdata rsync profile keys. A substitution
|
||||
# that only handles HOSTN_ leaves a live conf with a profile named hostn-appdata that nothing
|
||||
# references, which is what the pull script did before this flag existed.
|
||||
_target_slots=$(grep -oE '^[[:space:]]*HOST[0-9]+_' "$TARGET" 2>/dev/null \
|
||||
| grep -oE 'HOST[0-9]+' | sort -u)
|
||||
_target_slot=$(echo "$_target_slots" | head -1)
|
||||
if [[ $(echo "$_target_slots" | grep -c .) -gt 1 ]]; then
|
||||
echo "Error: '$TARGET' defines keys for more than one host slot:" >&2
|
||||
echo " $(echo "$_target_slots" | tr '\n' ' ')" >&2
|
||||
echo " A host conf owns exactly one slot. Refusing rather than picking one." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -n "$HOST_SLOT" ]]; then
|
||||
if ! [[ "$HOST_SLOT" =~ ^HOST[0-9]+$ ]]; then
|
||||
echo "Error: --host-slot must be HOST<n> (e.g. HOST1, HOST2) — got '$HOST_SLOT'" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$_target_slot" && "$_target_slot" != "$HOST_SLOT" ]]; then
|
||||
echo "Error: --host-slot says $HOST_SLOT but '$TARGET' defines ${_target_slot}_ keys." >&2
|
||||
echo " Substituting for the wrong slot removes every key the target actually has," >&2
|
||||
echo " credentials included. Refusing." >&2
|
||||
exit 1
|
||||
fi
|
||||
if grep -q -e 'HOSTN' -e 'hostn' "$TEMPLATE" 2>/dev/null; then
|
||||
_lower=$(echo "$HOST_SLOT" | tr '[:upper:]' '[:lower:]')
|
||||
_RESOLVED_TMPL="$(mktemp)"
|
||||
trap '[[ -n "${_RESOLVED_TMPL:-}" ]] && rm -f "$_RESOLVED_TMPL"' EXIT
|
||||
sed -e "s/HOSTN/${HOST_SLOT}/g" -e "s/hostn/${_lower}/g" "$TEMPLATE" > "$_RESOLVED_TMPL"
|
||||
TEMPLATE="$_RESOLVED_TMPL"
|
||||
echo " Resolved HOSTN → ${HOST_SLOT} for this host"
|
||||
fi
|
||||
elif grep -q 'HOSTN' "$TEMPLATE" 2>/dev/null; then
|
||||
# No slot declared and the template is still generic — the original refusal, unchanged.
|
||||
if [[ -n "$_target_slot" ]]; then
|
||||
_lower=$(echo "$_target_slot" | tr '[:upper:]' '[:lower:]')
|
||||
echo "Error: template still contains HOSTN placeholders, but the target uses ${_target_slot}_." >&2
|
||||
echo " Merging as-is would classify all ${_target_slot}_ keys as deprecated and remove" >&2
|
||||
echo " them — including every credential. Declare the slot:" >&2
|
||||
echo "" >&2
|
||||
echo " $0 --template $TEMPLATE --target $TARGET --host-slot ${_target_slot} --dry-run" >&2
|
||||
echo "" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Block-parsing patterns ───────────────────────────────────────────────────────────────────
|
||||
# All three walkers below share these. A single-line array — KEY=(a b c) — both opens and
|
||||
# closes on one line. Matching only the opener leaves the walker inside a block it never
|
||||
# leaves, so every key until the next standalone ")" becomes invisible: skipped by
|
||||
# _collect_stats, dropped from HOST_MAP by _parse_target, and emitted from the template
|
||||
# instead of the user's conf by _write_merged. _ARR_ONELINE_RE is what stops that.
|
||||
|
||||
_ARR_DECL_RE='^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\('
|
||||
_ARR_OPEN_RE='^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\('
|
||||
_ARR_CLOSE_RE='^[[:space:]]*\)[[:space:]]*(#.*)?$'
|
||||
# The [^#]* is deliberate: it forces the closing ")" to appear before any comment, so an
|
||||
# opener like FOO=( # see note (here) is not mistaken for a complete single-line array.
|
||||
_ARR_ONELINE_RE='=\([^#]*\)[[:space:]]*(#.*)?$'
|
||||
_SCALAR_RE='^[[:space:]]*([A-Z0-9_]+)[[:space:]]*='
|
||||
|
||||
# ── Parse target → KEY → full definition block ───────────────────────────────────────────────
|
||||
|
||||
declare -A HOST_MAP # KEY → complete definition line(s) from user's conf
|
||||
@@ -159,19 +275,23 @@ _parse_target() {
|
||||
if [[ "$in_block" == true ]]; then
|
||||
cur_block+="$line"$'\n'
|
||||
# Closing ) — optional trailing whitespace and comment
|
||||
if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
|
||||
if [[ "$line" =~ $_ARR_CLOSE_RE ]]; then
|
||||
HOST_MAP["$cur_key"]="$cur_block"
|
||||
in_block=false; cur_key=""; cur_block=""
|
||||
fi
|
||||
else
|
||||
# declare -A KEY=(
|
||||
if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
# KEY=(
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
# declare -A KEY=( / KEY=(
|
||||
if [[ "$line" =~ $_ARR_DECL_RE ]] || [[ "$line" =~ $_ARR_OPEN_RE ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"
|
||||
# KEY=(a b c) — opens and closes on one line, never enter block mode
|
||||
if [[ "$line" =~ $_ARR_ONELINE_RE ]]; then
|
||||
HOST_MAP["$cur_key"]="$line"$'\n'
|
||||
cur_key=""
|
||||
else
|
||||
in_block=true; cur_block="$line"$'\n'
|
||||
fi
|
||||
# KEY=value (simple scalar)
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
|
||||
elif [[ "$line" =~ $_SCALAR_RE ]]; then
|
||||
HOST_MAP["${BASH_REMATCH[1]}"]="$line"$'\n'
|
||||
fi
|
||||
fi
|
||||
@@ -188,7 +308,7 @@ _collect_stats() {
|
||||
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$in_block" == true ]]; then
|
||||
if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
|
||||
if [[ "$line" =~ $_ARR_CLOSE_RE ]]; then
|
||||
in_block=false
|
||||
TMPL_SEEN["$cur_key"]=1
|
||||
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
|
||||
@@ -196,11 +316,17 @@ _collect_stats() {
|
||||
cur_key=""
|
||||
fi
|
||||
else
|
||||
if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
|
||||
if [[ "$line" =~ $_ARR_DECL_RE ]] || [[ "$line" =~ $_ARR_OPEN_RE ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"
|
||||
if [[ "$line" =~ $_ARR_ONELINE_RE ]]; then
|
||||
TMPL_SEEN["$cur_key"]=1
|
||||
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
|
||||
else ADDED+=("$cur_key"); fi
|
||||
cur_key=""
|
||||
else
|
||||
in_block=true
|
||||
fi
|
||||
elif [[ "$line" =~ $_SCALAR_RE ]]; then
|
||||
local k="${BASH_REMATCH[1]}"
|
||||
TMPL_SEEN["$k"]=1
|
||||
if [[ -n "${HOST_MAP[$k]+_}" ]]; then KEPT+=("$k")
|
||||
@@ -223,18 +349,23 @@ _write_merged() {
|
||||
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||
if [[ "$in_block" == true ]]; then
|
||||
cur_block+="$line"$'\n'
|
||||
if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
|
||||
if [[ "$line" =~ $_ARR_CLOSE_RE ]]; then
|
||||
in_block=false
|
||||
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
|
||||
else printf '%s' "$cur_block"; fi
|
||||
cur_key=""; cur_block=""
|
||||
fi
|
||||
else
|
||||
if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
|
||||
elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
|
||||
if [[ "$line" =~ $_ARR_DECL_RE ]] || [[ "$line" =~ $_ARR_OPEN_RE ]]; then
|
||||
cur_key="${BASH_REMATCH[1]}"
|
||||
if [[ "$line" =~ $_ARR_ONELINE_RE ]]; then
|
||||
if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
|
||||
else printf '%s\n' "$line"; fi
|
||||
cur_key=""
|
||||
else
|
||||
in_block=true; cur_block="$line"$'\n'
|
||||
fi
|
||||
elif [[ "$line" =~ $_SCALAR_RE ]]; then
|
||||
local k="${BASH_REMATCH[1]}"
|
||||
if [[ -n "${HOST_MAP[$k]+_}" ]]; then printf '%s' "${HOST_MAP[$k]}"
|
||||
else printf '%s\n' "$line"; fi
|
||||
@@ -268,6 +399,24 @@ fi
|
||||
|
||||
echo " KEPT ${#KEPT[@]} existing vars — your values preserved"
|
||||
|
||||
# ── Guard: total mismatch ────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# Keeping nothing from a populated conf is never a real upgrade — it is the signature of the
|
||||
# two files not describing the same thing (wrong template, wrong target, unsubstituted
|
||||
# placeholders). The HOSTN check above catches the known cause; this catches the rest, because
|
||||
# the failure mode is silent and total: every value in the file is replaced by a default.
|
||||
# Deliberately fires in --dry-run too, so the report itself carries the warning.
|
||||
if [[ ${#KEPT[@]} -eq 0 && ${#REMOVED[@]} -gt 0 ]]; then
|
||||
echo "────────────────────────────────────────────────────────────────────────"
|
||||
echo ""
|
||||
echo "Error: refusing — this would keep NOTHING and remove all ${#REMOVED[@]} existing keys." >&2
|
||||
echo " A genuine upgrade preserves values; keeping zero means the template and the" >&2
|
||||
echo " target do not describe the same conf. Check that --template matches --target" >&2
|
||||
echo " and that any HOSTN placeholders were substituted for this host's slot." >&2
|
||||
echo "" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#ADDED[@]} -eq 0 && ${#REMOVED[@]} -eq 0 ]]; then
|
||||
echo " Already up to date — no changes needed."
|
||||
echo "────────────────────────────────────────────────────────────────────────"
|
||||
@@ -297,7 +446,7 @@ fi
|
||||
# Unraid /tmp is rootfs while the confs live on flash — a cross-device mv silently degrades
|
||||
# to copy-then-unlink, which is exactly the torn write this is meant to prevent.
|
||||
TMPOUT="$(mktemp "${TARGET}.XXXXXX")"
|
||||
trap 'rm -f "$TMPOUT"' EXIT
|
||||
trap 'rm -f "$TMPOUT"; [[ -n "${_RESOLVED_TMPL:-}" ]] && rm -f "$_RESOLVED_TMPL"' EXIT
|
||||
|
||||
# mktemp creates 0600; carry the target's existing mode/owner across so the installed conf
|
||||
# does not come back with different permissions than it went in with.
|
||||
@@ -316,4 +465,5 @@ fi
|
||||
# that constantly. A rename swaps the inode: readers get the old file or the new one.
|
||||
mv -f "$TMPOUT" "$TARGET"
|
||||
trap - EXIT
|
||||
[[ -n "$_RESOLVED_TMPL" ]] && rm -f "$_RESOLVED_TMPL"
|
||||
echo "Updated: $TARGET"
|
||||
|
||||
@@ -67,6 +67,7 @@
|
||||
# CERTIFICATE MONITOR domains checked for SSL expiry
|
||||
# SMART HEALTH drives to skip in SMART monitoring
|
||||
# ZFS REPORT pools to exclude from ZFS health report
|
||||
# PCIe AER QUIET dead PCIe devices removed at array start to stop AER log spam
|
||||
#
|
||||
# ── RESOURCE MANAGER ───────────────────────────────────────────────────────────────────────
|
||||
# RESOURCE MANAGER containers paused/stopped under memory pressure
|
||||
@@ -94,7 +95,7 @@
|
||||
|
||||
# ━━━ Identity ━━━
|
||||
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
|
||||
# SSH key used for all server-to-server operations — rsync, failover, conf sync.
|
||||
# SSH key used for all server-to-server operations — rsync, fallback, conf sync.
|
||||
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
|
||||
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
|
||||
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
|
||||
@@ -112,6 +113,11 @@
|
||||
HOSTN_EMBY_CONTAINER="Emby"
|
||||
HOSTN_EMBY_URL="http://localhost:8096"
|
||||
HOSTN_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
||||
HOSTN_EMBY_PUBLIC_URL="" # e.g. https://media.example.com/emby — browser-reachable base, used
|
||||
# to build image URLs that render in the WebGUI. Deliberately separate
|
||||
# from HOSTN_EMBY_URL: that one is for server-side API calls and is
|
||||
# usually localhost, which resolves to the wrong machine in a browser.
|
||||
# Empty = features that need an image quietly go without one.
|
||||
|
||||
# ━━━ Jellyfin ━━━
|
||||
HOSTN_JELLYFIN_CONTAINER="Jellyfin"
|
||||
@@ -123,6 +129,17 @@
|
||||
# Create in Gitea: Settings → Applications → Generate Token → scope: write:user
|
||||
HOSTN_GITEA_API_TOKEN=""
|
||||
|
||||
# ━━━ Bug Reports ━━━
|
||||
# Only used when BUG_REPORT_LOCAL_ENABLED=true in master.conf. Reports then go to this Gitea
|
||||
# instead of GitHub — and stay there, so they do not reach the Varaverk maintainer.
|
||||
#
|
||||
# Reached locally or over Tailscale, so the token never crosses the public proxy and no Authelia
|
||||
# bypass is needed. It is a credential and lives here rather than master.conf for that reason;
|
||||
# it is never shipped, and the settings UI masks it.
|
||||
HOSTN_BUG_REPORT_URL="" # e.g. http://gitea:3000 or the tailnet name
|
||||
HOSTN_BUG_REPORT_REPO="" # owner/repo
|
||||
HOSTN_BUG_REPORT_TOKEN="" # Gitea API token with issue-write on that repo
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# Discord webhook — leave blank to disable.
|
||||
HOSTN_DISCORD_WEBHOOK=""
|
||||
@@ -138,6 +155,23 @@
|
||||
# "Authelia|9091"
|
||||
)
|
||||
|
||||
# Shares rsynced to the mirror during onboard Step 1e, BEFORE the auth containers are created.
|
||||
# This is the only rsync an onboard performs — media is never seeded here.
|
||||
# Profile is inferred from the directory basename, so Critical-Data resolves to critical-data:
|
||||
# a clean copy with the auth containers stopped on both sides. Do not point this at a share
|
||||
# whose profile keeps databases running; a dirty copy of MariaDB or Redis is worse than none,
|
||||
# because the container starts, reads Up, and restarts a dead database behind it.
|
||||
HOSTN_PARTNERSHIP_PROVISION_SHARES=(
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — the auth stack
|
||||
)
|
||||
|
||||
# Containers that belong in "<PartnerShort>-Fallback" on the mirror rather than in a mirrored
|
||||
# copy of this host's folder layout — the ones that exist there only to cover this host going
|
||||
# dark. Everything else the onboard deploys is filed onto the same shelf it occupies here
|
||||
# (Arrs Stack, Networking, Databases…), because it runs on the mirror continuously.
|
||||
# Empty is the normal state: leave it empty unless a container is genuinely failover-only.
|
||||
HOSTN_PARTNERSHIP_FALLBACK_ONLY=()
|
||||
|
||||
# XML templates pushed to mirror during onboard — auth stack.
|
||||
# Dependencies (databases) must come before apps that depend on them.
|
||||
HOSTN_PARTNERSHIP_AUTH_STACK=(
|
||||
@@ -453,6 +487,8 @@
|
||||
HOSTN_RADARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated anime root
|
||||
HOSTN_RADARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Radarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Movies") — blank disables the download orphan cleaner for Radarr
|
||||
HOSTN_RADARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Radarr's container sees it (e.g. "/downloads/Completed/Movies") — needed to trigger import scans on held folders
|
||||
HOSTN_LIDARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Lidarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Music") — blank disables the download orphan cleaner for Lidarr
|
||||
HOSTN_LIDARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Lidarr's container sees it (e.g. "/downloads/Completed/Music") — needed to trigger import scans on held folders
|
||||
|
||||
declare -A HOSTN_RADARR_PATH_MAP=(
|
||||
# ["/movies"]="/mnt/user/Movies"
|
||||
@@ -510,6 +546,19 @@
|
||||
"sda" # boot USB — SMART not meaningful on flash drives
|
||||
)
|
||||
|
||||
# ━━━ PCIe AER Quiet ━━━
|
||||
# PCI addresses removed from the bus at array start so dead hardware stops spamming
|
||||
# correctable AER errors into syslog. Full DDDD:BB:DD.F form — find them with:
|
||||
# grep -o "from [0-9a-f:.]*" /var/log/syslog | sort | uniq -c | sort -rn
|
||||
#
|
||||
# Only devices bound to vfio-pci or to no driver at all are eligible. Anything with a
|
||||
# live driver is refused, so a mistyped address cannot pull an HBA or NIC out from
|
||||
# under a running system. Devices claimed by a running VM are refused too.
|
||||
# Gated by PCIE_QUIET_ENABLED in master.conf. Empty list = no-op.
|
||||
HOSTN_PCIE_QUIET_DEVICES=(
|
||||
# "0000:03:00.0"
|
||||
)
|
||||
|
||||
# ━━━ ZFS Report ━━━
|
||||
HOSTN_ZFS_REPORT_IGNORE_POOLS=(
|
||||
# "disk5"
|
||||
@@ -569,6 +618,27 @@
|
||||
HOSTN_LLDAP_USER="admin" # lldap admin username
|
||||
HOSTN_LLDAP_PASS="" # lldap admin password
|
||||
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Ollama / AI ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Per-host because only some nodes actually have a GPU. A node with an empty OLLAMA_URL is not
|
||||
# an error — it falls through to the resolver and uses another node's Ollama over Tailscale.
|
||||
|
||||
# ━━━ Ollama ━━━
|
||||
HOSTN_OLLAMA_URL="" # e.g. http://localhost:11434 — empty if no local Ollama
|
||||
HOSTN_OLLAMA_GPU_UUID="" # pins Ollama to one card on multi-GPU hosts
|
||||
HOSTN_OLLAMA_MODEL="hf.co/unsloth/Qwen3-14B-GGUF:IQ4_XS" # generation — must fully offload; see README-AI.md
|
||||
|
||||
# ━━━ Web search ━━━
|
||||
# Per-host because one is an address on this network and the other is a credential. Only the
|
||||
# General Chat profile can use these — it is the profile that cannot change anything, which is
|
||||
# why it is the one allowed to look outside. Off until AI_WEB_SEARCH_ENABLED says otherwise.
|
||||
HOSTN_DEGOOG_URL="" # e.g. http://localhost:4444 — self-hosted, no key, /api/search
|
||||
HOSTN_SEARXNG_URL="" # e.g. http://localhost:8888 — needs format: [json] in its settings.yml
|
||||
HOSTN_WEB_SEARCH_API_KEY="" # brave or tavily; unused when the provider is searxng
|
||||
HOSTN_OLLAMA_EMBED_MODEL="nomic-embed-text" # embeddings — the generation model cannot embed
|
||||
|
||||
# ━━━ Authelia ━━━
|
||||
HOSTN_AUTHELIA_CONFIG="/mnt/user/appdata-Fallback/Critical-Data/Authelia/configuration.yml"
|
||||
HOSTN_AUTHELIA_CONTAINER="Authelia"
|
||||
|
||||
+633
-53
@@ -115,12 +115,77 @@
|
||||
# scripts, state, and data are all available before the array mounts.
|
||||
# Both directories are created automatically if they don't exist.
|
||||
#
|
||||
# DATA_DIR — historical logs, statistics, discovery histories, blocklists
|
||||
# STATE_DIR — runtime state files for all scripts (watchdogs, fallback, transcode, etc.)
|
||||
# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no /boot/config root.
|
||||
# DATA_DIR is the one on-disk root. Everything Varaverk persists lives under it, in a
|
||||
# subdirectory named for what the files are. Move DATA_DIR and the whole tree follows.
|
||||
#
|
||||
# It used to be two roots plus two strays: DATA_DIR beside State_Files/ as siblings, with the
|
||||
# conf-cache backup off in SCRIPTS_DIR/.cache/vv/d and the arr cache backups loose in DATA_DIR's
|
||||
# root. Nothing was wrong with any one of those decisions; together they meant no single place
|
||||
# answered "what does Varaverk keep on disk". State is data — it is the data that happens to
|
||||
# describe right now — so it belongs under the same root as the rest.
|
||||
#
|
||||
# db/ — statistics, histories, counters, blocklists. Things that accumulate.
|
||||
# state/ — runtime state for every script: watchdogs, fallback, transcode, setup.
|
||||
# Requirement: ALL state files MUST use $STATE_DIR. No /tmp, no repo root.
|
||||
# ai/ — the retrieval index, operator memory, token ledger, filed bugs, saved chats.
|
||||
# cache/ — persistent backups of the tmpfs caches, and ONLY those. A file belongs here when
|
||||
# losing it costs a re-fetch and nothing else; anything that is a source of truth
|
||||
# belongs in db/ or state/.
|
||||
# logs/ — retained log output. Live logging still goes to LOG_DIR (/var/log/varaverk).
|
||||
#
|
||||
# STATE_DIR keeps its name and changes only its value, which is why this restructure did not
|
||||
# touch the 15 conf entries, 18 shell paths and 23 PHP paths that build on it.
|
||||
#
|
||||
# The tmpfs caches are NOT here and must not be moved here — see VV_CACHE_ROOT below. These are
|
||||
# on flash; those are read every second by the WebGUI and rewritten by the hundred megabytes.
|
||||
DATA_DIR="/boot/config/plugins/varaverk/data"
|
||||
STATE_DIR="/boot/config/plugins/varaverk/State_Files"
|
||||
PERSISTENT_CONF_CACHE="/boot/config/plugins/varaverk/.cache/vv/d"
|
||||
DB_DIR="${DATA_DIR}/db"
|
||||
STATE_DIR="${DATA_DIR}/state"
|
||||
AI_DATA_DIR="${DATA_DIR}/ai"
|
||||
CACHE_BACKUP_DIR="${DATA_DIR}/cache"
|
||||
LOG_ARCHIVE_DIR="${DATA_DIR}/logs"
|
||||
BACKUP_DIR="${DATA_DIR}/Backups"
|
||||
CONF_BACKUP_DIR="${BACKUP_DIR}/Confs" # 0700 — holds credentials
|
||||
PERSISTENT_CONF_CACHE="${CACHE_BACKUP_DIR}/conf"
|
||||
ARR_CACHE_BACKUP_DIR="${CACHE_BACKUP_DIR}/arr"
|
||||
|
||||
# ── Cache Roots ──
|
||||
# Everything Varaverk keeps in RAM, under one root, defined once.
|
||||
#
|
||||
# These used to be four literals in load_config.sh and three more in the PHP layer, spread over
|
||||
# /tmp/vv_cache, /tmp/arr_cache, /tmp/.cache/vv/d, /tmp/.cache/vv/ai and /tmp/varaverk_ai_jobs —
|
||||
# five naming schemes, and no single place that could tell you what Varaverk had in tmpfs. PHP
|
||||
# could not read load_config.sh, so it restated the paths it needed and the two layers were kept
|
||||
# in agreement by hand. They live here because master.conf is the one file both layers actually
|
||||
# read: bash sources it, and the PHP conf parser resolves ${VAR} against it the same way.
|
||||
#
|
||||
# ONE ROOT, NOT ONE DIRECTORY. The subdirectories are deliberately separate and must stay that
|
||||
# way — they have genuinely different rules:
|
||||
#
|
||||
# conf/ holds partner credentials, is chmod 700, and IS snapshotted to PERSISTENT_CONF_CACHE
|
||||
# so it survives a reboot.
|
||||
# ai/ holds non-secret token counters that are worthless when stale and must NOT be
|
||||
# preserved across a reboot — losing them means "not collected here" until the next
|
||||
# sync, which is the honest answer.
|
||||
# jobs/ holds in-flight work handed off to a detached worker — an AI answer being generated,
|
||||
# a container action being applied. chmod 700: each is readable by anyone who can guess
|
||||
# its token, which is why the tokens are random_bytes and not sequential.
|
||||
# arr/ is the large one (100s of MB) and is restored from a DATA_DIR backup on demand.
|
||||
# api/ is the WebGUI payload cache, and is the only one safe to delete at any moment.
|
||||
#
|
||||
# Collapsing those into a single directory would give the credentials the token cache's
|
||||
# persistence rules, or the reverse. The win here is one definition, not one folder.
|
||||
#
|
||||
# Point VV_CACHE_ROOT somewhere else and everything follows. It must be on a filesystem that is
|
||||
# cleared or safe to clear on boot — every consumer treats a missing cache as a cold start, and
|
||||
# nothing here is a source of truth for anything.
|
||||
VV_CACHE_ROOT="/tmp/varaverk"
|
||||
VV_CACHE_DIR="${VV_CACHE_ROOT}/api" # WebGUI payload cache — monitor, arrs, ai
|
||||
CONF_RAM_CACHE_DIR="${VV_CACHE_ROOT}/conf" # partner host*.conf — chmod 700, snapshotted
|
||||
ARR_CACHE_DIR="${VV_CACHE_ROOT}/arr" # arr payloads — restored from DATA_DIR on demand
|
||||
AI_TOKEN_CACHE_DIR="${VV_CACHE_ROOT}/ai" # partner token ledgers — never preserved
|
||||
AI_JOB_DIR="${VV_CACHE_ROOT}/jobs/ai" # in-flight AI answers — chmod 700
|
||||
DOCKER_JOB_DIR="${VV_CACHE_ROOT}/jobs/docker" # in-flight container actions — chmod 700
|
||||
|
||||
# ── Version Parity ──
|
||||
# Controls behaviour when local and remote unRAID versions differ.
|
||||
@@ -132,14 +197,21 @@
|
||||
# ── Conf Sync ──
|
||||
# conf_sync.sh pulls partner host confs into /tmp/.vv/ RAM cache on array start and after
|
||||
# any conf save — makes partner vars (HOST2_*, etc.) available without touching git.
|
||||
CONF_SYNC_ENABLED=true
|
||||
# Ships OFF for the same reason as RSYNC_ENABLED: a node seeded from this template has not yet
|
||||
# been paired with anyone, and pulling a partner's confs before the partnership exists is either
|
||||
# a no-op or a surprise. The onboard flow arms it once the pairing is real.
|
||||
CONF_SYNC_ENABLED=false
|
||||
|
||||
# ── Arr Sync ──
|
||||
# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
||||
# Runs before rsync — all nodes agree on tracked library before files are transferred.
|
||||
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
|
||||
ARR_SYNC_ENABLED=true
|
||||
ARR_SYNC_BLOCKLIST="${DATA_DIR}/arr_sync_blocklist.tsv"
|
||||
# Ships OFF. Bidirectional means a fresh node with an empty library is an equal party to the
|
||||
# merge, and "all nodes agree on tracked library" is not something you want decided by the node
|
||||
# that knows nothing. Onboard Step 9 runs arr_sync explicitly during bootstrap; this gate is for
|
||||
# the unattended runs afterwards.
|
||||
ARR_SYNC_ENABLED=false
|
||||
ARR_SYNC_BLOCKLIST="${DB_DIR}/arr_sync_blocklist.tsv"
|
||||
ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node
|
||||
ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches
|
||||
DOCKER_APPDATA_BASE="/mnt/user/appdata"
|
||||
@@ -201,8 +273,10 @@
|
||||
VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"
|
||||
|
||||
# Tailscale removal on offboard.
|
||||
PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard
|
||||
# false = skip removal (manual or testing)
|
||||
PARTNERSHIP_REMOVE_TAILSCALE=false # opt-in — removes the mirror from the tailnet on offboard.
|
||||
# OFF by default: it needs a TAILSCALE_API_KEY, which
|
||||
# expires and must be reissued, and a feature that
|
||||
# silently no-ops on a stale key should not be opt-out.
|
||||
|
||||
# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
|
||||
# Stays in shared conf — only owner uses it, and owner is always running this script.
|
||||
@@ -273,7 +347,16 @@
|
||||
GITEA_CONTAINER="Gitea"
|
||||
GITEA_REPO_PATH="" # e.g. YourUser/Varaverk.git
|
||||
GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS
|
||||
TARGET_DIR="/mnt/user/appdata/Varaverk"
|
||||
# Derived, never a literal. This is the directory git_pull_execute.sh pulls into, and it is the
|
||||
# one host-specific path that used to live in master.conf as a fixed string — which made the
|
||||
# shared conf unshareable: pushing it to a partner that had migrated its install repointed that
|
||||
# partner's pull target at the old location, and the next pull built a second copy there while
|
||||
# the live install ran somewhere else.
|
||||
#
|
||||
# SCRIPTS_DIR is exported by load_config.sh from its own location before this file is sourced, so
|
||||
# this resolves to wherever the install actually is, on every node, in either storage mode, with
|
||||
# nothing to keep in step. DATA_DIR and STATE_DIR below already worked this way.
|
||||
TARGET_DIR="${SCRIPTS_DIR}"
|
||||
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
||||
SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222)
|
||||
GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh
|
||||
@@ -289,16 +372,18 @@
|
||||
# Scripts launched by array_started.sh when the array comes online.
|
||||
# Launched in order — each as a background process.
|
||||
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||||
# Continuous scripts (failover) run until array stops.
|
||||
# Continuous scripts (fallback) run until array stops.
|
||||
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
|
||||
# watchdog_orchestrator.sh — NOT launched here.
|
||||
ARRAY_START_SCRIPTS=(
|
||||
"Plugin/unraid/System_Essentials/pcie_aer_quiet.sh" # drop AER-spamming dead hardware — runs first so later logs stay readable
|
||||
"Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register Varaverk API key — registry is ephemeral
|
||||
"System_Essentials/conf_sync.sh" # pull partner confs + push own conf into /tmp/.vv/ RAM cache
|
||||
"System_Essentials/conf_cache_restore.sh" # load partner confs from persistent backup if conf_sync couldn't reach partner
|
||||
"Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts
|
||||
"unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill
|
||||
"unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||||
"unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||||
"System_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill
|
||||
"Plugin/unraid/System_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning
|
||||
"System_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted
|
||||
"Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
|
||||
"Arrs_Stack/arr_cache_prefill.sh" # warm Lidarr/Sonarr/Radarr tracked-data caches before anything reads them cold
|
||||
"Arrs_Stack/start_webhook_listener.sh" # arr upgrade webhook listener — continuous
|
||||
@@ -311,10 +396,10 @@
|
||||
# Order matters: user scripts first (prevents new ops), then data movement, then containers.
|
||||
ARRAY_STOP_SCRIPTS=(
|
||||
"System_Essentials/conf_cache_save.sh" # snapshot partner conf RAM cache → /boot before anything stops
|
||||
"unRAID_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops
|
||||
"Plugin/unraid/System_Essentials/user_scripts_stop.sh" # stop background scripts before they start new ops
|
||||
"Fallback/fallback.sh --stop" # gracefully stop fallback (not caught by user_scripts_stop)
|
||||
"unRAID_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below)
|
||||
"unRAID_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files)
|
||||
"System_Essentials/rsync_stop.sh --rsync-only" # kill rsync; skip container recovery (handled below)
|
||||
"Plugin/unraid/System_Essentials/mover_stop.sh" # stop mover after rsync (they conflict on same files)
|
||||
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
||||
)
|
||||
|
||||
@@ -338,8 +423,14 @@
|
||||
"Watchdogs/resource_watchdog.sh" # reduce system pressure before healing attempts
|
||||
"Watchdogs/docker_watchdog.sh" # heal containers with freed resources
|
||||
"Watchdogs/system_watchdog.sh" # system component health — storage + webgui
|
||||
"Plugin/unraid/System_Essentials/unraid_api_key_renew.sh" # re-register API key if registry lost it
|
||||
"Plugin/unraid/Tools/ai_repair_sweep.sh" # read what the last cycle logged; off unless AI_REPAIR_ENABLED
|
||||
"Watchdogs/stability_watchdog.sh" # reboot if all else fails — last line of defense
|
||||
)
|
||||
# The repair sweep sits ahead of stability deliberately, which is the one exception to "stability
|
||||
# last". It reads the previous cycle's logs and may correct the very misconfiguration stability
|
||||
# would otherwise reboot for — a wrong port is not fixed by restarting the machine. It is bounded
|
||||
# by AI_PROBE_TIMEOUT, exits 0 in every case, and does nothing at all unless AI_REPAIR_ENABLED.
|
||||
|
||||
# ━━━ System Watchdog ━━━
|
||||
# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
|
||||
@@ -399,9 +490,9 @@
|
||||
"Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files
|
||||
"Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files
|
||||
# Daily, not weekly — DOWNLOAD_ORPHAN_AGE gates each folder on its own mtime, so
|
||||
# frequency costs nothing in safety, but a weekly run clears a whole week of
|
||||
# newly-eligible orphans at once and can exceed DOWNLOAD_ORPHAN_MAX_DELETE_GB, which
|
||||
# aborts the pass entirely and rolls the backlog into an even larger next run.
|
||||
# frequency costs nothing in safety, while a weekly run makes each pass a whole
|
||||
# week of newly-eligible orphans at once and far more likely to spill past
|
||||
# DOWNLOAD_ORPHAN_MAX_DELETE_GB into a deferred remainder.
|
||||
"Arrs_Stack/arr_download_orphan_cleaner.sh" # sweep orphaned completed downloads out of the SAB Completed folders — deletes junk + already-imported leftovers, triggers import scans for genuinely-missing content
|
||||
# Runs after cleanup, not before — frees disk space from orphans/junk first, so the
|
||||
# searches triggered below (for relocated hasFile=false/episodeFileCount=0 entries)
|
||||
@@ -412,8 +503,24 @@
|
||||
"Arrs_Stack/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||
"Arrs_Stack/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||
"Arrs_Stack/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||
# Before the two docker jobs below, deliberately: they pull images and restart
|
||||
# containers, so a container can legitimately not exist for a moment while they run.
|
||||
# Reading docker in its settled overnight state keeps a rebuild from costing a strike.
|
||||
# Three strikes at this cadence means three days genuinely absent before conf is edited.
|
||||
"Tools/conf_container_prune.sh" # drop containers that no longer exist from the conf lists
|
||||
"Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
||||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||||
# Last, and after git_pull_execute.sh which must always run first. Daily rather than with
|
||||
# the Sunday cert monitor: CERT_HISTORY_STRIKES counts passes, so a weekly cadence would
|
||||
# make five strikes five weeks. One NPM list call; it is what gives the Certs tab its
|
||||
# per-domain renewal and failure counts.
|
||||
"Plugin/unraid/Tools/cert_history.sh" # record cert renewals, failures and age per domain
|
||||
"Plugin/unraid/Tools/npm_access_stats.sh" # aggregate NPM per-host access logs into request and byte totals
|
||||
# After both of the above, because it reads what they write — the uptime history and the
|
||||
# per-host request totals are two of the four things it reasons from. Daily rather than
|
||||
# hourly: everything it files is a condition that has already lasted hours by the time
|
||||
# AUTH_SWEEP_DOWN_MIN lets it through, so a faster cadence would find nothing new.
|
||||
"Plugin/unraid/Tools/auth_sweep.sh" # file findings for hosts that are not serving, and hostnames Authelia is not protecting
|
||||
)
|
||||
|
||||
# Pull latest images for DAILY_RESTART_CONTAINERS before the daily restart.
|
||||
@@ -439,9 +546,9 @@
|
||||
# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am)
|
||||
WEEKLY_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/docker_update.sh --weekly" # pull latest images for WEEKLY_RESTART_CONTAINERS before restart
|
||||
"Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync
|
||||
"Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers
|
||||
"unRAID_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority
|
||||
"System_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority
|
||||
"Arrs_Stack/arr_full_rescan.sh" # full disk↔DB reconciliation for Lidarr/Sonarr/Radarr — keeps tracked stats honest, runs before discovery so it works off fresh data
|
||||
"Arrs_Stack/arr_corruption_scan.sh --remediate" # ffprobe-based corruption sweep of Sonarr's tracked files — deletes+re-searches only after CORRUPTION_SCAN_STRIKE_LIMIT consecutive hits on the same file
|
||||
"Arrs_Stack/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history
|
||||
@@ -466,8 +573,8 @@
|
||||
# than restarting them a second time. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS is
|
||||
# treated as untrustworthy (docker_update.sh likely didn't run, or didn't run recently) — deleted,
|
||||
# and every container in that tier restarts normally, same as if the file never existed.
|
||||
DOCKER_UPDATE_REBUILT_DAILY_FILE="$DATA_DIR/docker_update_rebuilt_daily.list"
|
||||
DOCKER_UPDATE_REBUILT_WEEKLY_FILE="$DATA_DIR/docker_update_rebuilt_weekly.list"
|
||||
DOCKER_UPDATE_REBUILT_DAILY_FILE="${DB_DIR}/docker_update_rebuilt_daily.list"
|
||||
DOCKER_UPDATE_REBUILT_WEEKLY_FILE="${DB_DIR}/docker_update_rebuilt_weekly.list"
|
||||
DOCKER_UPDATE_REBUILT_STALE_HOURS=12
|
||||
|
||||
# Shares synced during the weekly maintenance window — defined per host in host*.conf.
|
||||
@@ -507,6 +614,7 @@
|
||||
"Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot
|
||||
"Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp
|
||||
"Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains
|
||||
"Monitors/uptime_report.sh" # domains down now, and any that were not 100% this week
|
||||
"Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums
|
||||
"Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown
|
||||
"Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio
|
||||
@@ -528,7 +636,7 @@
|
||||
# Fine grained control — disable specific orchestrators while keeping others
|
||||
# Use when: rebuilding secondary, testing, per-window bandwidth management
|
||||
#
|
||||
# Example — HOST2 data rebuild:
|
||||
# Example — rebuilding a partner's data:
|
||||
# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
||||
# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs
|
||||
# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild
|
||||
@@ -536,15 +644,37 @@
|
||||
# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe)
|
||||
# MONTHLY_RSYNC_ENABLED=true ← monthly_maintenance.sh rsync section
|
||||
# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works
|
||||
# MEDIA_SEED_ENABLED=false ← the onboard first-fill never starts
|
||||
#
|
||||
# MEDIA_SEED_ENABLED is the odd one out: it does not gate a schedule, it gates a single
|
||||
# multi-week transfer. Rsync/media_seed.sh pushes every DAILY_SYNC_SHARES entry to a newly
|
||||
# onboarded partner — on HOST1 that is ~28 TB against BW_LIMIT, so weeks. Turn it off when the
|
||||
# partner is going to be filled some other way (a physically moved disk, an existing library),
|
||||
# and onboard will finish without ever starting it. Turning it back on does not start anything
|
||||
# by itself; dispatch it from the Partnership tab or run the script.
|
||||
#
|
||||
# NOTE ON THE DEFAULT: this one ships OFF, and unlike Tier 1 it is not armed by a successful
|
||||
# onboard — Step 9c arms RSYNC_ENABLED, CONF_SYNC_ENABLED and ARR_SYNC_ENABLED, and
|
||||
# deliberately not this. Seeding a partner is a decision about weeks of bandwidth and tens of
|
||||
# terabytes of disk, and the node that has just been onboarded is exactly the node nobody has
|
||||
# checked the free space on yet. Arm it when you have decided the partner should be filled by
|
||||
# rsync rather than by hand. Turning it on is one switch; noticing it was on is not.
|
||||
# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
|
||||
# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true
|
||||
RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
||||
# NOTE ON THE DEFAULT: Tier 1 ships OFF. This template is what a brand-new node is seeded from
|
||||
# (Manual-Deployment.md — "Rebuilding a Wiped Node"), and a node that has just been stood up has
|
||||
# empty shares and an unverified partner. Defaulting the global gate ON meant a fresh install
|
||||
# started transferring — with --delete in the mix — before anyone had looked at it. Tier 2 stays
|
||||
# ON so the structure is intact the moment you do arm Tier 1; this mirrors HOST1's live posture.
|
||||
# Changing this default cannot affect an existing node: conf_upgrade KEEPS values already set.
|
||||
RSYNC_ENABLED=false # Tier 1 — global gate, overrides everything below
|
||||
CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh rsync section
|
||||
INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section
|
||||
DAILY_RSYNC_ENABLED=true # Tier 2 — daily_sync_maintenance.sh rsync section
|
||||
WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
||||
MONTHLY_RSYNC_ENABLED=true # Tier 2 — monthly_maintenance.sh rsync section
|
||||
FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback
|
||||
MEDIA_SEED_ENABLED=false # Tier 2 — Rsync/media_seed.sh, the onboard first-fill
|
||||
|
||||
# ━━━ Download Webhook ━━━
|
||||
# Immediate push to remote nodes on every Sonarr/Radarr/Lidarr Download event.
|
||||
@@ -568,6 +698,13 @@
|
||||
# lowered from 23 on 2026-07-16 — daily maintenance jobs alone now
|
||||
# take ~4h; 19h cap leaves room for them to still run same-day
|
||||
# before the next 1am fire even if a share hits the cap
|
||||
RSYNC_MIN_FREE_GB=50 # warn if any disk backing the remote share is under this
|
||||
# (0 disables). Online is not writable: a full destination
|
||||
# makes rsync stall rather than fail. Warns, never aborts.
|
||||
CONF_PRUNE_STRIKE_LIMIT=3 # consecutive runs a container must be absent from docker
|
||||
# before its name is removed from the conf lists. A
|
||||
# rebuild makes a container vanish briefly; one miss
|
||||
# must never be enough to edit conf.
|
||||
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
||||
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||||
@@ -670,7 +807,25 @@
|
||||
declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
[arrs_stack]="logs *.tmp"
|
||||
[critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
||||
[critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
|
||||
# appendonlydir and mariadb-bin.* are excluded for the same reason as *.db-wal: they are
|
||||
# written continuously by a database this profile copies WHILE IT RUNS. The exclusion had
|
||||
# SQLite's WAL and nothing for Redis or MariaDB, and both broke the mirror on 2026-08-17.
|
||||
#
|
||||
# The failure is not a torn file, it is a surviving pointer to a missing one. The manifest
|
||||
# and the index are a few hundred bytes and land instantly; the data files they name are
|
||||
# large and are still in flight. The mirror ends up holding a promise nothing can keep:
|
||||
# Redis appendonly.aof.manifest → appendonly.aof.30.base.rdb (absent) → exit 1
|
||||
# MariaDB mariadb-bin.index, 265 → mariadb-bin.000265 (absent) → Aborting
|
||||
#
|
||||
# Excluded rather than repaired: neither is needed for the engine to START. Redis rebuilds
|
||||
# an empty AOF, MariaDB begins a new binlog sequence. On a fallback mirror that is the
|
||||
# right trade — those files exist for persistence and replication, and a database that
|
||||
# starts empty beats one that cannot start at all.
|
||||
#
|
||||
# NOT a guarantee that a live-copied InnoDB is consistent. It happened to survive both
|
||||
# times. The honest fix is a DB-layer dump synced as a file; this stops the observed
|
||||
# breakage without pretending to be that.
|
||||
[critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm appendonlydir mariadb-bin.*"
|
||||
[important-data]="logs *.tmp"
|
||||
[emby]="logs transcodes cache crash*"
|
||||
)
|
||||
@@ -705,7 +860,34 @@
|
||||
|
||||
EXTERNAL_IP="8.8.8.8"
|
||||
FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
|
||||
FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s)
|
||||
# Cost measured 2026-08-22: ~15ms CPU and 784 bytes per cycle
|
||||
# per host (~2 MB/day, half of it across the mesh). The 2s of
|
||||
# wall time per cycle is ping waiting, not work.
|
||||
FALLBACK_HANDBACK_STRIKES=4 # consecutive healthy checks before initiating handback
|
||||
# 4×30s = 2min. Raising this costs nothing — it requires more
|
||||
# of the checks that already happen, it does not add any.
|
||||
# Any failed check resets the counter, so the window is a
|
||||
# genuine "stayed up this long", not "was up this often".
|
||||
# Entry is deliberately unguarded — one false from ping_remote enters FALLBACK, with no strike
|
||||
# count, because every second of a real outage is downtime and DDNS TTL is racing it. The
|
||||
# confidence comes from the probe itself instead: see FALLBACK_PROBE_* below and _ping_retry()
|
||||
# in common.sh.
|
||||
FALLBACK_PROBE_RETRIES=3 # reachability attempts before declaring a host down
|
||||
FALLBACK_PROBE_RETRY_DELAY=2 # seconds between those attempts
|
||||
# Success returns on the first try, so a healthy cycle costs
|
||||
# nothing extra. Only a failure pays: 3×4s + 2×2s = 16s worst
|
||||
# case, which must stay under FALLBACK_CHECK_INTERVAL.
|
||||
FALLBACK_DRY_RUN=false # run the daemon in PREVIEW mode, permanently
|
||||
# Survives array start, unlike a --dry-run typed at a shell:
|
||||
# array_started.sh launches with no arguments, so a hand-run
|
||||
# preview is replaced by the LIVE daemon at the next boot.
|
||||
# Decides nothing, starts nothing, moves no DNS, sends no
|
||||
# notification — it only reports what it WOULD have done.
|
||||
# Set true to observe real outages before arming for real.
|
||||
FALLBACK_LOG_MAX_MB=5 # size cap for data/logs/fallback.log before one rotation
|
||||
# Event-only output (the default, no --log) is a few lines
|
||||
# per incident and holds years; --log fills this in about a
|
||||
# fortnight and keeps the previous one as fallback.log.1
|
||||
FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
|
||||
FALLBACK_ENABLED=false # set true once both servers are configured and paired
|
||||
# false = suppresses "not running" warnings in status scripts
|
||||
@@ -822,7 +1004,7 @@
|
||||
# Restart loop protection — prevents watchdog from endlessly restarting a broken container
|
||||
WATCHDOG_CONTAINER_RESTART_LIMIT=3
|
||||
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours
|
||||
WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
|
||||
WATCHDOG_CONTAINER_RESTART_LOG="${DB_DIR}/container_restart_history.db"
|
||||
|
||||
# Notification batching — one summary per cycle instead of one ping per event
|
||||
WATCHDOG_BATCH_NOTIFY=true
|
||||
@@ -898,7 +1080,7 @@
|
||||
# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary.
|
||||
INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit
|
||||
PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children
|
||||
TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db"
|
||||
TUNING_MONITOR_LOG="${DB_DIR}/system_tuning_history.db"
|
||||
TUNING_LOG_RETENTION=30 # days before old entries are purged
|
||||
|
||||
# ━━━ Reboot ━━━
|
||||
@@ -920,6 +1102,19 @@
|
||||
# log lines per hour that have no diagnostic value. Filter removes them at source.
|
||||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||||
|
||||
# ━━━ PCIe AER Quiet ━━━
|
||||
# Master gate for pcie_aer_quiet.sh — removes dead PCIe hardware from the bus at
|
||||
# array start so it stops flooding syslog with correctable AER errors.
|
||||
#
|
||||
# Correctable means the link recovered, so the errors are harmless — but the kernel
|
||||
# logs every one. Removing the device ends it at the source. Unlike pci=noaer this
|
||||
# keeps uncorrectable AER reporting alive on every other device, which the AI repair
|
||||
# triage relies on to tell a real fault from this noise.
|
||||
#
|
||||
# Devices are listed per host in host*.conf as HOST*_PCIE_QUIET_DEVICES.
|
||||
# Off by default — turn on only after filling in that list for this host.
|
||||
PCIE_QUIET_ENABLED=false
|
||||
|
||||
# ━━━ PHP-FPM ━━━
|
||||
# Higher max_children allows more concurrent PHP requests to the unRAID WebGUI.
|
||||
# Default is very low — increasing it prevents WebGUI slowdowns under load.
|
||||
@@ -1072,17 +1267,19 @@
|
||||
LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this
|
||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||||
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
|
||||
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
|
||||
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
||||
# lidarr_missing_art.sh, lidarr_release_fixer.sh, and arr_cache_prefill.sh. See
|
||||
# lidarr_get_tracked_data() in common.sh for the fresh/stale/rescan-active branching logic.
|
||||
LIDARR_CACHE_FILE="$DATA_DIR/lidarr_tracked_cache.json"
|
||||
LIDARR_RESCAN_DURATION_DB="$DATA_DIR/lidarr_rescan_duration.db"
|
||||
LIDARR_CACHE_FILE="${ARR_CACHE_BACKUP_DIR}/lidarr_tracked_cache.json"
|
||||
LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db"
|
||||
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
|
||||
ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr
|
||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma" "wv" "ape" "aiff" "aif" "alac" "dsf" "dff" "mpc" "tta")
|
||||
LIDARR_PROTECTED_PATTERNS=(
|
||||
# Metadata
|
||||
"*.nfo" "*.tbn"
|
||||
@@ -1101,7 +1298,7 @@
|
||||
LIDARR_ART_RETRIES=2 # download retry attempts per image
|
||||
LIDARR_ART_SLEEP_BETWEEN=0.2 # seconds between fanart.tv API calls
|
||||
LIDARR_ART_RECHECK_DAYS=30 # days before re-querying art that upstream didn't have
|
||||
LIDARR_ART_MISS_CACHE="${DATA_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had
|
||||
LIDARR_ART_MISS_CACHE="${DB_DIR}/lidarr_art_miss_cache.tsv" # negative cache — art upstream has never had
|
||||
# HOST*_FANART_API_KEY / HOST*_LASTFM_API_KEY — set in host*.conf
|
||||
|
||||
# Lidarr discovery settings (playback_aware_lidarr_discovery.sh)
|
||||
@@ -1111,7 +1308,7 @@
|
||||
LIDARR_DISCOVERY_USER_CAP_PCT=35 # max % any single user can contribute to play score (prevents one listener dominating)
|
||||
LIDARR_DISCOVERY_MAX_ADDS=5 # max artists to add per run — quality over bulk
|
||||
LIDARR_DISCOVERY_REJECT_COOLDOWN=30 # days before re-evaluating a rejected artist
|
||||
LIDARR_DISCOVERY_HISTORY="$DATA_DIR/lidarr_discovery_history.db"
|
||||
LIDARR_DISCOVERY_HISTORY="${DB_DIR}/lidarr_discovery_history.db"
|
||||
|
||||
# Sonarr discovery settings (playback_aware_sonarr_discovery.sh)
|
||||
SONARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100)
|
||||
@@ -1123,7 +1320,7 @@
|
||||
SONARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected show
|
||||
SONARR_DISCOVERY_USER_EPISODE_CAP=8 # max episodes any one user contributes to seed volume score
|
||||
SONARR_DISCOVERY_MONITOR_MODE="all" # Sonarr monitor mode on add: all | future | first | latest | none
|
||||
SONARR_DISCOVERY_HISTORY="$DATA_DIR/sonarr_discovery_history.db"
|
||||
SONARR_DISCOVERY_HISTORY="${DB_DIR}/sonarr_discovery_history.db"
|
||||
|
||||
# Radarr discovery shared settings
|
||||
RADARR_DISCOVERY_THRESHOLD=52 # score to accept candidate (0-100) — lower than Lidarr since diverse seeds rarely overlap
|
||||
@@ -1134,7 +1331,7 @@
|
||||
RADARR_DISCOVERY_MIN_RATING=60 # min TMDB vote_average × 10 (60 = 6.0/10)
|
||||
RADARR_DISCOVERY_REJECT_COOLDOWN=60 # days before re-evaluating a rejected movie
|
||||
RADARR_DISCOVERY_SEED_LIBRARIES=("Movies") # Emby libraries to draw seed movies from
|
||||
RADARR_DISCOVERY_HISTORY="$DATA_DIR/radarr_discovery_history.db"
|
||||
RADARR_DISCOVERY_HISTORY="${DB_DIR}/radarr_discovery_history.db"
|
||||
|
||||
# Emby → arr sync library allowlists
|
||||
# Only these Emby library names will be considered by the sync tools.
|
||||
@@ -1147,19 +1344,30 @@
|
||||
SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this
|
||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
SONARR_TRACKED_COUNT_FILE="$DATA_DIR/sonarr_tracked.count"
|
||||
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
|
||||
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
|
||||
# reach "completed" — generous because a large series can sit
|
||||
# queued behind other moves already in progress, not just its
|
||||
# own copy time
|
||||
CORRUPTION_SCAN_STATE_FILE="${DATA_DIR}/corruption_scan_state.tsv" # clean-file skip-cache
|
||||
CORRUPTION_SCAN_STRIKES_FILE="${DATA_DIR}/corruption_scan_strikes.tsv" # consecutive corrupt-detection counts, keyed by host path
|
||||
CORRUPTION_SCAN_STATE_FILE="${DB_DIR}/corruption_scan_state.tsv" # clean-file skip-cache
|
||||
CORRUPTION_SCAN_STRIKES_FILE="${DB_DIR}/corruption_scan_strikes.tsv" # consecutive corrupt-detection counts, keyed by host path
|
||||
CORRUPTION_SCAN_STRIKE_LIMIT=2 # consecutive corrupt detections (across separate scan runs)
|
||||
# required before --remediate deletes+re-searches — guards
|
||||
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
|
||||
# triggering an unnecessary delete. Resets to 0 the moment a
|
||||
# file probes clean again.
|
||||
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
|
||||
# newly-scanned files reads as corrupt. A healthy library sits
|
||||
# near zero; a high rate means the detector is wrong, not the
|
||||
# library. Only counts ffprobe exit != 0.
|
||||
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
|
||||
# cleanly — catches the ffprobe container dying mid-scan,
|
||||
# which the pre-flight health check cannot see.
|
||||
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
|
||||
# ceiling is not applied — too small a sample to judge.
|
||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||
SONARR_PROTECTED_PATTERNS=(
|
||||
# Subtitles
|
||||
@@ -1185,7 +1393,13 @@
|
||||
RADARR_MAX_DELETE_GB=30 # require --i-know-what-im-doing if deletion exceeds this
|
||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
RADARR_TRACKED_COUNT_FILE="$DATA_DIR/radarr_tracked.count"
|
||||
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
|
||||
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
|
||||
# removed. Gate 6 already catches an API returning far too few
|
||||
# tracked files; this catches the partial failure too small to trip
|
||||
# that percentage — one root folder failing to enumerate makes its
|
||||
# movies look orphaned, and a transient one will not repeat.
|
||||
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
|
||||
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
|
||||
@@ -1206,10 +1420,17 @@
|
||||
# Per-host dirs: HOST*_SONARR_DOWNLOAD_DIR / HOST*_RADARR_DOWNLOAD_DIR (+ _CONTAINER_DIR).
|
||||
DOWNLOAD_ORPHAN_CLEANER_ENABLED=true
|
||||
DOWNLOAD_ORPHAN_AGE=7 # days — entries younger than this may be mid-import, never touched
|
||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives)
|
||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # abort delete pass over this — a partial queue fetch would classify
|
||||
# live downloads as orphans, and a huge total is that failure's symptom;
|
||||
# --i-know-what-im-doing overrides for known backlogs
|
||||
DOWNLOAD_ORPHAN_KEEP_MARKER=".vv-keep" # a file by this name inside a download folder pins it —
|
||||
# never classified, never deleted. For lossless rips the
|
||||
# library holds only as MP3, which REDUNDANT would sweep
|
||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives) — Sonarr/Radarr
|
||||
DOWNLOAD_ORPHAN_MIN_AUDIO_MB=2 # same test for Lidarr — a 50M floor would mark every album folder
|
||||
# as junk, since single tracks rarely reach it
|
||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # per-run delete budget — a backlog above this drains
|
||||
# safest-first (JUNK, REDUNDANT, then UNMATCHED) up to
|
||||
# the budget and defers the rest to the next run.
|
||||
# Live downloads are protected by DOWNLOAD_ORPHAN_AGE,
|
||||
# not by this; --i-know-what-im-doing clears it in one pass
|
||||
|
||||
# ━━━ Arr Content Classification (radarr/sonarr_classification_scan.sh) ━━━
|
||||
#
|
||||
@@ -1339,7 +1560,7 @@
|
||||
|
||||
# Daily statistics log — read by weekly_health_digest.sh for transcode summary.
|
||||
TRANSCODE_STATE_FILE="$STATE_DIR/transcode_state.db"
|
||||
TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
|
||||
TRANSCODE_DAILY_LOG="${DB_DIR}/transcode_daily.db"
|
||||
TRANSCODE_LOG_RETENTION=90 # days before old entries purged
|
||||
|
||||
TRANSCODE_CHECK_EMBY=true
|
||||
@@ -1352,6 +1573,23 @@
|
||||
# Defined per host in host*.conf — Emby container names and keys differ per server:
|
||||
# HOST1_TRANSCODE_SERVERS / HOST2_TRANSCODE_SERVERS
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AUTH STACK ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Auth Stack ━━━
|
||||
# Which identity stack this mesh runs behind its protected hostnames. The Auth tab reads this to
|
||||
# decide which panels to draw and which endpoints to call — it is a routing switch, not a
|
||||
# migration. Changing it does not move users, groups or rules between stacks; stand the new one
|
||||
# up first, then point this at it.
|
||||
#
|
||||
# Mesh-wide rather than per host: auth is the one service the partnership treats as shared, owned
|
||||
# by the mesh owner and consumed by everyone else, so two nodes disagreeing about which stack is
|
||||
# in force would mean two different answers to "who is this person".
|
||||
# "authelia_lldap" — Authelia for access rules, lldap for users and groups. Fully implemented.
|
||||
# "authentik" — single stack for both. NOT IMPLEMENTED YET; the tab says so instead of drawing panels that cannot work.
|
||||
AUTH_STACK="authelia_lldap"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MONITORS ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -1365,6 +1603,62 @@
|
||||
CERT_CRIT_DAYS=7 # critical alert within this many days
|
||||
CERT_TIMEOUT=10 # seconds per domain before giving up
|
||||
|
||||
# ── Certificate History ──
|
||||
# Tools/cert_history.sh records every certificate NPM holds — first seen, renewals, failures —
|
||||
# into DB_DIR/cert_history.json, which is what the Certs tab draws its per-domain cards from.
|
||||
# A domain found expired on this many consecutive passes stops being counted as working. It is
|
||||
# not deleted: the history of something that broke is the reason the file is kept.
|
||||
CERT_HISTORY_STRIKES=5 # expired passes before a domain is retired from the active list
|
||||
|
||||
# ── Uptime Probe ──
|
||||
# Tools/uptime_probe.sh checks every hostname NPM serves, once a minute, from outside the proxy.
|
||||
# NPM's access log only describes hosts somebody visited; this is what watches the ones nobody
|
||||
# does — which is where a quietly broken container hides.
|
||||
#
|
||||
# A 302 to the Authelia portal or a 401 counts as UP. The question is whether the server is there,
|
||||
# and an auth redirect is proof that it is; counting it as down would mark every protected host on
|
||||
# this mesh permanently offline.
|
||||
#
|
||||
# Probes carry the User-Agent Varaverk-Uptime/1.0, which npm_access_stats.sh excludes from its
|
||||
# request counts. Fifty thousand self-inflicted requests a day would otherwise bury real traffic.
|
||||
UPTIME_PROBE_ENABLED=true # master switch for the per-minute probe
|
||||
UPTIME_PROBE_TIMEOUT=8 # seconds per domain before it counts as down
|
||||
UPTIME_PROBE_LIST_TTL=300 # seconds to reuse the domain list from NPM before re-reading it
|
||||
|
||||
# ── Auth Sweep ──
|
||||
# Tools/auth_sweep.sh asks the two questions the Auth tab answers about one host, about every host,
|
||||
# and files what it finds as findings on the AI tab. Reports only — it starts nothing and rewrites
|
||||
# nothing, because every remedy here (start a container, edit a rule, change a default policy) is a
|
||||
# decision rather than a correction.
|
||||
#
|
||||
# proxy_down a host below UPTIME_MIN that has been failing longer than DOWN_MIN. The time gate
|
||||
# is what keeps a reboot from filing a finding for every hostname on the machine.
|
||||
# access_open an Authelia instance whose default policy lets through every hostname its rules do
|
||||
# not decide. Filed once per instance, not once per hostname — they all have the
|
||||
# same single fix, and one finding per name is twenty-two copies of one sentence.
|
||||
#
|
||||
# The access half reads whichever Authelia each proxy host actually points at, which is not always
|
||||
# the one HOST*_AUTHELIA_CONFIG names — this installation runs two.
|
||||
AUTH_SWEEP_ENABLED=true # master switch
|
||||
AUTH_SWEEP_UPTIME_MIN=96 # 24h percentage below which a host becomes a candidate
|
||||
AUTH_SWEEP_DOWN_MIN=120 # minutes it must have been failing before anything is filed
|
||||
AUTH_SWEEP_ACCESS_CHECK=true # run the "is it actually protected" half at all
|
||||
|
||||
# ── Cert Triage ──
|
||||
# Tools/cert_triage.sh reads certbot's own logs and names why renewals failed. cert_history.sh
|
||||
# counts failures by noticing an expiry in the past; this reads the reason.
|
||||
#
|
||||
# Counts runs, not lines: one log file is one certbot invocation, and one failure writes its
|
||||
# reason into the ACME response, the traceback and certbot's summary, so line counting reports it
|
||||
# three times and inflates whichever category is most verbose.
|
||||
#
|
||||
# The log directory is found from the NPM container's own mount. Set CERT_TRIAGE_LOG_DIR only if
|
||||
# that lookup cannot work. Both bounds exist because this is reachable from a page request and the
|
||||
# directory here is 639 MB across a thousand rotated files.
|
||||
CERT_TRIAGE_FILES=40 # rotated logs to read, newest first by rotation suffix
|
||||
CERT_TRIAGE_MAX_BYTES=262144 # bytes read from the end of each — a run's reason is always last
|
||||
CERT_TRIAGE_LOG_DIR="" # empty = find it from the NPM container
|
||||
|
||||
# ━━━ Backup Verify ━━━
|
||||
# Verifies rsync mirror health by comparing random file checksums between servers.
|
||||
# Catches silent corruption or incomplete syncs that rsync itself wouldn't detect.
|
||||
@@ -1372,6 +1666,10 @@
|
||||
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
||||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
|
||||
# scales with size (~50MB/s); this caps it. A fixed
|
||||
# connect-sized timeout killed multi-GB checksums and
|
||||
# the empty result was then reported as MISSING.
|
||||
|
||||
# ━━━ SMART Health ━━━
|
||||
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||||
@@ -1384,7 +1682,7 @@
|
||||
|
||||
# ━━━ ZFS Memory Snapshot ━━━
|
||||
# Weekly ZFS pool health and memory diagnostic report — informational only.
|
||||
ZFS_REPORT_LOG="/var/log/zfs-weekly-health.log"
|
||||
ZFS_REPORT_LOG="${LOG_ARCHIVE_DIR}/zfs-weekly-health.log"
|
||||
ZFS_REPORT_ARC_WARN_PCT=90 # warn if ARC using more than this % of its max
|
||||
ZFS_REPORT_ARC_FREE_WARN_GB=10 # warn if ARC headroom (max - current) drops below this GB
|
||||
ZFS_REPORT_AVAIL_WARN_GB=20 # warn if less than this GB available on ZFS pool
|
||||
@@ -1395,15 +1693,15 @@
|
||||
# ━━━ Bandwidth Monitor ━━━
|
||||
# Called automatically by rsync.sh after each sync — one bounded write per run.
|
||||
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
|
||||
BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
|
||||
BANDWIDTH_LOG="${DB_DIR}/bandwidth_history.db"
|
||||
BANDWIDTH_LOG_RETENTION=90 # days before old entries purged
|
||||
BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report
|
||||
|
||||
# Stats files — written by cleanup and recovery scripts, read by coffee report.
|
||||
# All in DATA_DIR — array always running when these are written.
|
||||
ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
|
||||
ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats
|
||||
ARR_RECOVERY_FAILURE_COUNTS="$DATA_DIR/arr_recovery_failure_counts.db" # per-item chronic-failure tracking
|
||||
ARR_CLEANUP_STATS="${DB_DIR}/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
|
||||
ARR_RECOVERY_STATS="${DB_DIR}/arr_recovery_stats.db" # blocklist + re-search stats
|
||||
ARR_RECOVERY_FAILURE_COUNTS="${DB_DIR}/arr_recovery_failure_counts.db" # per-item chronic-failure tracking
|
||||
|
||||
# ━━━ Health Digest ━━━
|
||||
# Aggregated system health summary — reads existing state files, no new writes.
|
||||
@@ -1513,9 +1811,17 @@
|
||||
SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
|
||||
DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
|
||||
DOCKER_WATCHDOG_INTENTIONAL_FILE="$STATE_DIR/docker_intentional_stops.db"
|
||||
WATCHDOG_MUTE_FILE="$STATE_DIR/watchdog_mutes.db"
|
||||
SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
|
||||
SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"
|
||||
|
||||
# ━━━ Timed Mutes ━━━
|
||||
# A mute silences one container for a bounded time and then expires on its own. It exists because
|
||||
# every other exemption here is permanent — SCAN_IGNORE is a conf edit, an intentional stop lasts
|
||||
# until cleared — so a temporary problem gets a permanent workaround that nobody revisits.
|
||||
# Capped so "temporary" is enforced rather than intended: a mute longer than this is refused.
|
||||
WATCHDOG_MUTE_MAX_HOURS=8
|
||||
|
||||
# ━━━ Strike and Reboot Loop Settings ━━━
|
||||
# Strike system: a check must fail this many consecutive cycles before action is taken.
|
||||
# Single spikes (one bad reading) are ignored — sustained problems trigger reboot.
|
||||
@@ -1593,6 +1899,280 @@
|
||||
SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity better than crashing mid-check
|
||||
SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move better than crashing mid-move
|
||||
|
||||
# ==============================================================================================
|
||||
# ── AI / RAG ──────────────────────────────────────────────────────────────────────────────────
|
||||
# ━━━ Conf Backups ━━━
|
||||
# Every write through the plugin copies the conf aside first, to CONF_BACKUP_DIR with an
|
||||
# ISO-8601 stamp. The confs are gitignored, so that directory is the whole recovery path — there
|
||||
# is no history to revert to. A backup that cannot be taken cancels the write.
|
||||
# Oldest are pruned past this count, per conf file.
|
||||
CONF_BACKUP_RETAIN=30
|
||||
|
||||
# ==============================================================================================
|
||||
# Varaverk works exactly as well with AI off as with it on. Nothing below is required for any
|
||||
# script to function — every feature that can lean on AI has a complete non-AI path, and the
|
||||
# AI path is an enhancement layered on top. If Ollama is unreachable, callers proceed without it.
|
||||
#
|
||||
# AI_ENABLED is necessary but not sufficient. Every feature stays individually off until it has
|
||||
# earned it — narration for months before anything is allowed near a decision.
|
||||
|
||||
# ━━━ AI Master Switch ━━━
|
||||
# Which node runs the model, the retrieval index and the bug store. Every other node borrows it
|
||||
# over the mesh, so a box without a GPU still gets the assistant — it just does not get the AI tab.
|
||||
# Defaults to host1 when unset or malformed: whoever builds the mesh is host1.
|
||||
AI_OWNER_HOST="host1"
|
||||
|
||||
# Fail-closed: anything other than the literal "true" means off.
|
||||
AI_ENABLED=false
|
||||
AI_CONNECT_TIMEOUT=5 # seconds — probe when resolving which node has Ollama
|
||||
AI_REQUEST_TIMEOUT=240 # seconds — must clear a cold model load
|
||||
AI_RESOLVE_CACHE_TTL=300 # seconds — don't re-probe the mesh every invocation
|
||||
AI_MAX_RETRIES=1 # AI is enhancement; do not retry hard
|
||||
|
||||
# ━━━ AI Retrieval Index ━━━
|
||||
# The RAG index over this repo's own headers and documentation. Regenerable in minutes and
|
||||
# gitignored — it is derived data, never a source of truth.
|
||||
#
|
||||
# Only git-tracked files are ever indexed. Configurations/, State_Files/ and data/ are
|
||||
# gitignored, which is what makes it structurally impossible for a credential to reach the
|
||||
# index: the files holding them were never in the repo. Do not "improve" this to a filesystem
|
||||
# walk — an embedded secret cannot be rotated out of a vector.
|
||||
AI_INDEX_DB="${AI_DATA_DIR}/ai_index.db"
|
||||
AI_INDEX_BATCH=32 # chunks per embed request
|
||||
# A pull is the only thing that changes tracked files on a server, so it is the only moment the
|
||||
# index can go stale — and staleness is invisible in the answers, which keep citing the old
|
||||
# text with full confidence. Incremental: unchanged files are skipped, a no-op run is ~66ms.
|
||||
# Also gated on AI_ENABLED, and ai_index.sh refuses on its own unless that is true.
|
||||
AI_INDEX_ON_PULL=true # re-index after a git pull that changed tracked files
|
||||
AI_SEARCH_K=8 # chunks retrieved per query
|
||||
AI_SEARCH_PER_FILE=3 # cap per file so one document cannot fill the context
|
||||
|
||||
# ━━━ AI Memory ━━━
|
||||
# A small operator-maintained file the assistant is given at the start of every conversation:
|
||||
# who you are, how this install is set up, decisions already made, things it should stop asking.
|
||||
#
|
||||
# Injected into the prompt, never indexed. It lives under DATA_DIR, which is gitignored — that
|
||||
# is deliberate and load-bearing. Indexing it would embed a file that changes constantly, and
|
||||
# vector similarity is the wrong way to retrieve "things I was told to remember"; it also keeps
|
||||
# personal notes out of a repository that gets pushed.
|
||||
#
|
||||
# The character cap is a context budget, not a style guide. At 16384 the retrieved passages,
|
||||
# the model's reasoning and the conversation history are already competing; memory takes its
|
||||
# share off the top of every single turn, so keep it short and factual.
|
||||
# Two slots, and which one a fact lands in decides how much authority it carries.
|
||||
#
|
||||
# assisted written by the operator. The prompt tells the model to prefer it over retrieved
|
||||
# passages — a human asserting a fact about their own machine outranks a doc that
|
||||
# may be stale. This is the file the AI tab edits.
|
||||
# learned proposed by the assistant, accepted by the operator. A hint only: retrieval
|
||||
# overrules it, and it is trimmed first when the budget bites. Deliberately the
|
||||
# weaker seat, because memory the model writes AND the prompt ranks above the source
|
||||
# code would let one wrong conclusion restate itself forever.
|
||||
#
|
||||
# AI_MEMORY_FILE is the pre-split name and is still honoured: while the assisted file does not
|
||||
# exist, the legacy path is read instead, so an upgrade loses nothing.
|
||||
AI_MEMORY_ASSISTED_FILE="${AI_DATA_DIR}/mem_assisted.md"
|
||||
AI_MEMORY_LEARNED_FILE="${AI_DATA_DIR}/mem_learned.md"
|
||||
AI_MEMORY_FILE="${AI_DATA_DIR}/ai_memory.md" # legacy — read only if the assisted file is absent
|
||||
|
||||
# Which profiles each slot is given. "*" is all of them. Narrowing is how a General Chat question
|
||||
# about bash syntax stops carrying this machine's PCIe topology and disk serials.
|
||||
AI_MEMORY_ASSISTED_PROFILES="*"
|
||||
AI_MEMORY_LEARNED_PROFILES="*"
|
||||
|
||||
AI_MEMORY_MAX_CHARS=4000 # ~1000 tokens — the budget for both slots together
|
||||
# A ceiling on the learned slot alone, well under the total. A store that grows on its own would
|
||||
# otherwise end up occupying the whole budget, and the operator's own memory is what would get
|
||||
# truncated away — the exact inversion of which one matters.
|
||||
AI_MEMORY_LEARNED_MAX_CHARS=1200
|
||||
|
||||
# ━━━ AI Learned Memory ━━━
|
||||
# Lets the assistant propose facts worth keeping. It never writes memory directly: a candidate is
|
||||
# filed in data/ai/mem_proposals/ and the operator accepts or dismisses it, exactly as findings
|
||||
# work. Accepted text lands in the learned slot, which the prompt ranks BELOW retrieval.
|
||||
#
|
||||
# The reason for the indirection: memory rides on every future prompt. A model allowed to write
|
||||
# its own — and told to prefer it over the passages — would restate a wrong conclusion forever,
|
||||
# reading its own claim back as evidence. A bad proposal must cost one dismissal, not that.
|
||||
#
|
||||
# Costs nothing while off: the instruction that asks for a candidate is only added to the prompt
|
||||
# when this is true, so a disabled feature is genuinely absent rather than merely ignored.
|
||||
AI_MEMORY_LEARN_ENABLED=false
|
||||
|
||||
# Writes accepted candidates without asking. Cannot outrank its parent — with proposing off this
|
||||
# does nothing. Leave it false until the proposals have proven good for a while.
|
||||
AI_MEMORY_LEARN_AUTO_ACCEPT=false
|
||||
|
||||
# ━━━ AI Stored Conversations ━━━
|
||||
# How many past conversations the AI tab and the Monitor tab's AI row keep. One JSON file per
|
||||
# conversation under DATA_DIR/ai_chats, saved automatically when a turn completes; the oldest
|
||||
# drop off once the count is exceeded.
|
||||
#
|
||||
# A cap rather than a retention age. These are read by picking one out of a short list, and a
|
||||
# list you have to scroll is a list you stop using — the useful window is the handful of things
|
||||
# you were recently working on, which is a count, not a date.
|
||||
#
|
||||
# Not indexed and never retrieved into a prompt on their own. A stored chat only re-enters the
|
||||
# model's context when the operator explicitly reopens it, and it is re-validated per message on
|
||||
# the way in, exactly as live history is.
|
||||
AI_CHAT_HISTORY_MAX=10 # conversations kept — clamped to 1-50
|
||||
|
||||
# ━━━ AI Token Accounting ━━━
|
||||
# One row per completed turn, appended by whichever path ran it — the WebGUI worker and the
|
||||
# ai_query.sh CLI both write here, so the totals are not silently the tab's alone. Format is
|
||||
# pipe-delimited to match the other data/*.db files:
|
||||
#
|
||||
# date|time|host|profile|source|prompt_tokens|completion_tokens|tok_s
|
||||
#
|
||||
# The host column records where the turn ran, not where it is read. Each host keeps its own
|
||||
# data/ and nothing syncs it, so a host only ever sees its own rows — the column is there so
|
||||
# the file is already shaped right if the partner payload fetch is ever extended to carry it.
|
||||
# A column added later cannot be backfilled.
|
||||
#
|
||||
# Retention is by row count rather than age: pruning is considered only when the file passes a
|
||||
# size threshold, so an ordinary turn costs one stat() and an append.
|
||||
AI_TOKEN_DB="${AI_DATA_DIR}/ai_token_history.db"
|
||||
AI_TOKEN_RETAIN_ROWS=20000 # oldest rows dropped past this — years of ordinary use
|
||||
|
||||
# AI/ai_token_sync.sh pulled each partner's ledger into a tmpfs cache so the fleet total was a
|
||||
# fleet total. The mesh shares one AI now: every turn generates on the owner and is recorded
|
||||
# there against the node that asked, so the owner's own ledger already holds the whole fleet and
|
||||
# a partner ledger is empty by construction. Left off the schedule rather than deleted — the
|
||||
# reader still merges any cache from before the change, which describes real turns.
|
||||
AI_TOKEN_SYNC_ENABLED=false
|
||||
|
||||
|
||||
# ━━━ AI Feature Toggles ━━━
|
||||
# Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script
|
||||
# already made. Tier 3 assists a human. Enable in that order, and give each one weeks.
|
||||
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
|
||||
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
|
||||
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
|
||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
|
||||
# classification in the log; decides nothing. Off = identical deletions.
|
||||
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
|
||||
# nightly cleanup is not an assist — it is silently skipped past this.
|
||||
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
|
||||
|
||||
# ━━━ AI Repair ━━━
|
||||
# Reads the log of a job that has finished, turns known error shapes into findings, and offers
|
||||
# or applies a repair. Two switches, because detecting and repairing are separate things to
|
||||
# trust.
|
||||
#
|
||||
# AI_REPAIR_ENABLED on its own reads, files findings and offers fixes, and writes nothing. Run
|
||||
# it there first — long enough to read what it finds and disagree with some of it.
|
||||
#
|
||||
# AI_REPAIR_AUTOFIX_ENABLED is what allows a value to be written unattended, and only ever a
|
||||
# value a probe has answered on. Never a toggle: whether something should be switched on is a
|
||||
# decision about intent, and a probe cannot prove intent the way it can prove a port answers.
|
||||
AI_REPAIR_ENABLED=false
|
||||
AI_REPAIR_AUTOFIX_ENABLED=false
|
||||
|
||||
# ━━━ AI Repair Findings ━━━
|
||||
# Misconfigurations found in this installation, as opposed to defects in Varaverk — those go to
|
||||
# ai_bugs. A finding is open until the configuration is right, and closes itself when the probe
|
||||
# that proved the fault starts passing. Closed ones are kept for a while, because "this happened
|
||||
# before and here is what fixed it" is worth more than the disk. Open findings are never pruned:
|
||||
# an unresolved problem does not stop mattering because it is old.
|
||||
AI_FINDING_RETAIN_DAYS=90
|
||||
# Seconds a single probe may take. Nothing is written to conf that has not answered a probe, so
|
||||
# this is the budget for proving a candidate — kept short because a sweep may try several, and
|
||||
# an address worth switching to answers quickly or is not worth switching to.
|
||||
AI_PROBE_TIMEOUT=4
|
||||
# Tell the operator when a finding needs them. Only findings the sweep cannot resolve itself are
|
||||
# announced, one notification for all of them rather than one each, and each is announced once —
|
||||
# it stays quiet through every later pass until the fault changes or gets worse. Delivery is
|
||||
# whatever notify() is set up to use: NOTIFY_UNRAID and the host's Discord webhook.
|
||||
#
|
||||
# On by default, unlike the two switches above. Those gate reading and writing, which are things
|
||||
# to be trusted first. This gates telling someone, which is the reason for having looked.
|
||||
AI_REPAIR_NOTIFY_ENABLED=true
|
||||
|
||||
# ━━━ AI Repair: what it reads ━━━
|
||||
# Beyond Varaverk's own logs and the arrs' health endpoints.
|
||||
#
|
||||
# The system log catches what Varaverk cannot see about itself — a disk throwing I/O errors, a
|
||||
# filesystem going read-only, a PCIe link retraining every two minutes. Container restarts and
|
||||
# OOM kills are deliberately left to docker_watchdog, which already notifies on them.
|
||||
#
|
||||
# Container logs catch the opposite blind spot: docker_watchdog watches whether a container is
|
||||
# up and answering, which a container that has been unable to write to its database all day
|
||||
# passes perfectly. Only environment faults are matched — disk full, read-only filesystem,
|
||||
# corrupt database, fd limits, expired certificates — because those strings come from libc, the
|
||||
# kernel and SQLite and mean the same thing in all fifty containers. Anything app-specific
|
||||
# belongs in that app's own health endpoint.
|
||||
#
|
||||
# Both are bounded by time (since the last pass) and by a line cap, so a flood costs one pass.
|
||||
# Tools/ai_log_check.sh replays this host's real logs against the patterns — run it after
|
||||
# changing any of them.
|
||||
AI_REPAIR_SYSLOG_ENABLED=true
|
||||
AI_REPAIR_SYSLOG_MAX_LINES=4000
|
||||
AI_REPAIR_CONTAINER_LOGS_ENABLED=true
|
||||
AI_REPAIR_CONTAINER_LOG_LINES=400
|
||||
|
||||
# ━━━ AI Web Search ━━━
|
||||
# General Chat only. Every other profile either reads this installation or changes it; chat holds
|
||||
# no capability at all, and that is exactly why searching is the one thing it may do — a read that
|
||||
# leaves the house is safe on the profile that cannot act on what it finds. The Varaverk assistant
|
||||
# deliberately does not get it: its contract is that answers come from this installation's own
|
||||
# documents, and a web result there is an answer that looks sourced and is not.
|
||||
#
|
||||
# Off by default, and not because it is dangerous. Searching sends the operator's question to
|
||||
# something outside this house, and that is their decision to make rather than a default to
|
||||
# inherit. Nothing turns it on.
|
||||
#
|
||||
# Asked for per turn as well — there is a checkbox on the AI tab, and a question is only searched
|
||||
# when it is ticked. A question about this machine hands off to the assistant before the search
|
||||
# would run, so it never reaches the internet even with the box ticked.
|
||||
#
|
||||
# Provider: degoog | searxng | brave | tavily
|
||||
# degoog self-hosted, no key — set HOST*_DEGOOG_URL. Aggregates several engines and returns
|
||||
# them merged. The default, and the only one verified against a live service here
|
||||
# searxng self-hosted, no key, no third party — set HOST*_SEARXNG_URL, and enable format: [json]
|
||||
# in its own settings.yml, which the default image ships with off
|
||||
# brave HOST*_WEB_SEARCH_API_KEY, free tier available
|
||||
# tavily HOST*_WEB_SEARCH_API_KEY
|
||||
AI_WEB_SEARCH_ENABLED=false
|
||||
AI_WEB_SEARCH_PROVIDER=degoog
|
||||
AI_WEB_SEARCH_RESULTS=4
|
||||
AI_WEB_SEARCH_TIMEOUT=6
|
||||
|
||||
# ━━━ AI Chat Caution ━━━
|
||||
# General Chat cannot see this installation, and web search gives it confident-sounding material
|
||||
# about the outside world. The danger is the overlap: a question about THIS machine, answered
|
||||
# from a general page about Unraid, reads exactly like an answer about this machine.
|
||||
#
|
||||
# Detection is grammatical rather than topical — possessives, "this box", state questions, "what
|
||||
# happened last night" — because the topics are unbounded and the grammar is not. When it fires,
|
||||
# chat keeps its caution and defers instead of answering from the web.
|
||||
#
|
||||
# Pipe-separated extra phrases for what grammar misses: a nickname for the box, a share name,
|
||||
# anything that in practice means "mine". Matched as literal text, not as patterns.
|
||||
AI_CHAT_MY_SYSTEM_PHRASES=""
|
||||
|
||||
# ━━━ Bug Reports ━━━
|
||||
# A bug the assistant files is written here and goes nowhere until the owner sends it. Nothing is
|
||||
# ever transmitted automatically — the report is shown in full, read-only, and sending is a
|
||||
# second, separate press.
|
||||
#
|
||||
# Two destinations, and they are not a fallback chain. Fetching code from several mirrors is
|
||||
# harmless because they all serve the same thing; sending a report is not, because the
|
||||
# destinations are different people. Local ships OFF so an install that has configured nothing
|
||||
# reports upstream rather than silently into a tracker nobody reads.
|
||||
#
|
||||
# LOCAL ON — reports go to your own Gitea (see HOSTN_BUG_REPORT_* in host*.conf) and stay there.
|
||||
# They do NOT reach the Varaverk maintainer. Turn it on if you want your own backlog.
|
||||
# LOCAL OFF — reports open a prefilled GitHub issue you submit under your own account.
|
||||
BUG_REPORT_LOCAL_ENABLED=false
|
||||
BUG_REPORT_GITHUB_REPO="FailedProxy/Varaverk"
|
||||
|
||||
# ━━━ AI Conf Write Access ━━━
|
||||
# Separate switch from AI_ENABLED, off by default, and an explicit key whitelist. Never paths,
|
||||
# never credentials, never a container name. An empty whitelist means no writes regardless of
|
||||
# the toggle.
|
||||
AI_CONF_WRITE_ENABLED=false
|
||||
AI_CONF_WRITE_KEYS=()
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
Executable
+310
@@ -0,0 +1,310 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================== DATA LAYOUT MIGRATION =========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# One-time move of everything Varaverk persists into a single rooted tree under DATA_DIR.
|
||||
#
|
||||
# State_Files/ → data/state/
|
||||
# data/*.db|.count|.tsv|.list → data/db/
|
||||
# data/ai_* → data/ai/
|
||||
# data/*_tracked_cache.json → data/cache/arr/
|
||||
# data/*.log → data/logs/
|
||||
# SCRIPTS_DIR/.cache/vv/d/ → data/cache/conf/
|
||||
#
|
||||
# ==============================================================================================
|
||||
# WHY THIS EXISTS SEPARATELY FROM conf_upgrade
|
||||
# ==============================================================================================
|
||||
#
|
||||
# conf_upgrade adds keys the template has and the installation does not; it never rewrites a
|
||||
# value the operator already has, which is exactly the behaviour you want from it and exactly
|
||||
# why it cannot perform this migration. The paths being moved are existing keys — STATE_DIR,
|
||||
# BANDWIDTH_LOG, AI_INDEX_DB and two dozen more — so their values would keep pointing at the old
|
||||
# layout forever while the new directory variables sat beside them unused.
|
||||
#
|
||||
# So this rewrites those values, then moves the files to match. Both halves, or neither: a conf
|
||||
# pointing at a directory the data is not in is worse than not having started.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Two halves, in order: move the files, then rewrite the conf keys that point at them. Doing it
|
||||
# the other way round would leave every path variable naming a location nothing had reached yet,
|
||||
# and any script that ran in between would create the old layout again underneath the new one.
|
||||
#
|
||||
# Idempotent. A path already under DATA_DIR is left alone, so a re-run after a partial migration
|
||||
# finishes the job rather than moving things twice or failing on what is already done.
|
||||
#
|
||||
# One-time by intent, not by a marker file. There is no "already migrated" flag — the check is
|
||||
# whether each individual path is already where it belongs, which is also what makes an
|
||||
# interrupted run safe to repeat.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Existing keys are rewritten, which is why conf_upgrade cannot do this.
|
||||
# conf_upgrade adds keys the template has and the installation does not, and never rewrites a
|
||||
# value the operator already holds — correct for it, and exactly why it is the wrong tool here.
|
||||
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB and two dozen more are existing keys whose values must
|
||||
# change, or they would go on naming the old layout forever while the new directory variables
|
||||
# sat beside them unused.
|
||||
#
|
||||
# Move, never copy-and-hope.
|
||||
# The data being relocated is the only copy — statistics, histories, the AI index, arr caches.
|
||||
# Everything is moved and the source is gone afterwards, so there is no second location that
|
||||
# might still be written to by something that missed the change.
|
||||
#
|
||||
# The conf rewrite is the last thing, and the riskiest thing.
|
||||
# Until it happens the installation still works from the old layout. That ordering means an
|
||||
# abort partway through leaves a system that runs, rather than one whose paths point at
|
||||
# nothing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Idempotent
|
||||
# Every step tests before acting. A second run reports "already migrated" and changes nothing,
|
||||
# which matters because the natural instinct after a partial failure is to run it again.
|
||||
#
|
||||
# Moves, never copies-and-deletes
|
||||
# mv within one filesystem is atomic per file, so a reader either sees the file at the old
|
||||
# path or the new one — never a half-written copy at both. Nothing is deleted; if a file
|
||||
# cannot be moved it is reported and left exactly where it is.
|
||||
#
|
||||
# Conf is backed up before it is rewritten
|
||||
# master.conf.bak-<stamp>, next to the original, same convention conf_upgrade uses.
|
||||
#
|
||||
# Refuses to run while the orchestrators might be writing
|
||||
# A watchdog that sourced conf before the rewrite and writes state after the move would put a
|
||||
# file back at the old path. The window is seconds and the damage is one stale file, but the
|
||||
# check costs nothing and the failure is silent otherwise.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# This script reads conf to find the old locations and rewrites conf to record the new ones. It
|
||||
# is the one script here whose purpose is to change these values rather than obey them.
|
||||
#
|
||||
# Read to locate what moves
|
||||
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB, AI_MEMORY_FILE, AI_TOKEN_DB, ARR_CLEANUP_STATS,
|
||||
# ARR_SYNC_BLOCKLIST, CORRUPTION_SCAN_STATE_FILE, LIDARR_CACHE_FILE, ZFS_REPORT_LOG and the
|
||||
# rest of the per-script path keys — roughly two dozen in total.
|
||||
#
|
||||
# Written as the new roots
|
||||
# DATA_DIR and the directories beneath it: DB_DIR, STATE_DIR, AI_DATA_DIR,
|
||||
# CACHE_BACKUP_DIR, ARR_CACHE_BACKUP_DIR, CONF_CACHE_BACKUP_DIR, LOG_ARCHIVE_DIR.
|
||||
#
|
||||
# Every rewritten value is expressed as ${DB_DIR}/… rather than an absolute path, so a later
|
||||
# storage-mode migration moves them again by changing one variable.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# migrate_data_layout.sh --dry-run Show what would move. Changes nothing. Do this first.
|
||||
# migrate_data_layout.sh Perform the migration.
|
||||
# migrate_data_layout.sh --force Skip the running-orchestrator check.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
CONF="$ROOT/Configurations/master.conf"
|
||||
|
||||
DRY_RUN=false
|
||||
FORCE=false
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
--force) FORCE=true ;;
|
||||
*) echo "Unknown argument: $a" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -f "$CONF" ]] || { echo "[FATAL] master.conf not found at $CONF" >&2; exit 1; }
|
||||
|
||||
# Resolve the roots the same way load_config.sh will after this runs.
|
||||
SCRIPTS_DIR="$ROOT"
|
||||
DATA_DIR="$(grep -m1 -E '^\s*DATA_DIR=' "$CONF" | cut -d'"' -f2)"
|
||||
DATA_DIR="${DATA_DIR//\$\{SCRIPTS_DIR\}/$SCRIPTS_DIR}"
|
||||
DATA_DIR="${DATA_DIR:-$ROOT/data}"
|
||||
|
||||
OLD_STATE="$ROOT/State_Files"
|
||||
OLD_CONFCACHE="$ROOT/.cache/vv/d"
|
||||
|
||||
DB_DIR="$DATA_DIR/db"
|
||||
STATE_DIR="$DATA_DIR/state"
|
||||
AI_DATA_DIR="$DATA_DIR/ai"
|
||||
CACHE_BACKUP_DIR="$DATA_DIR/cache"
|
||||
ARR_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/arr"
|
||||
CONF_CACHE_BACKUP_DIR="$CACHE_BACKUP_DIR/conf"
|
||||
LOG_ARCHIVE_DIR="$DATA_DIR/logs"
|
||||
|
||||
moved=0; skipped=0; failed=0
|
||||
|
||||
say() { printf ' %s\n' "$*"; }
|
||||
step() { printf '\n━━━ %s ━━━\n' "$*"; }
|
||||
|
||||
# ── Guard: orchestrators mid-run ──────────────────────────────────────────────
|
||||
# Scoped to THIS installation's path, not to the script names. pgrep is system-wide, and a box
|
||||
# running both a production checkout and a development clone will always have one of them busy —
|
||||
# matching on "Orchestrators/" alone made a dev migration abort because prod was mid-cycle, which
|
||||
# is a process that cannot touch this tree's data at all. The full path is what distinguishes
|
||||
# them, so that is what is matched.
|
||||
if [[ "$FORCE" == false && "$DRY_RUN" == false ]]; then
|
||||
running=$(pgrep -fa "$ROOT/(Orchestrators|Watchdogs|Media|Rsync)/" 2>/dev/null \
|
||||
| grep -v "$$" | grep -v migrate_data_layout || true)
|
||||
if [[ -n "$running" ]]; then
|
||||
echo "[ABORT] A job from this installation is running — it may rewrite state mid-move:" >&2
|
||||
echo "$running" >&2
|
||||
echo "Wait for it to finish, or re-run with --force if you are sure." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Move one path ─────────────────────────────────────────────────────────────
|
||||
move() {
|
||||
local src="$1" dstdir="$2" base
|
||||
base="$(basename "$src")"
|
||||
[[ -e "$src" ]] || return 0
|
||||
if [[ -e "$dstdir/$base" ]]; then
|
||||
say "skip $base — already at ${dstdir#$DATA_DIR/}/"
|
||||
((skipped++)); return 0
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
say "would $base → ${dstdir#$DATA_DIR/}/"
|
||||
((moved++)); return 0
|
||||
fi
|
||||
mkdir -p "$dstdir" 2>/dev/null
|
||||
if mv "$src" "$dstdir/$base" 2>/dev/null; then
|
||||
say "moved $base → ${dstdir#$DATA_DIR/}/"
|
||||
((moved++))
|
||||
else
|
||||
say "FAILED $base — left in place"
|
||||
((failed++))
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 1. Rewrite the conf values conf_upgrade cannot ────────────────────────────
|
||||
step "Step 1: master.conf path values"
|
||||
if grep -q 'STATE_DIR="\${DATA_DIR}/state"' "$CONF"; then
|
||||
say "already migrated — no conf changes needed"
|
||||
else
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
cp "$CONF" "${CONF}.bak-$(date +%Y%m%d%H%M%S)"
|
||||
sed -i -E \
|
||||
-e 's|^(\s*STATE_DIR=)".*"|\1"${DATA_DIR}/state"|' \
|
||||
-e 's|^(\s*PERSISTENT_CONF_CACHE=)".*"|\1"${CACHE_BACKUP_DIR}/conf"|' \
|
||||
"$CONF"
|
||||
for v in ARR_SYNC_BLOCKLIST DOCKER_UPDATE_REBUILT_DAILY_FILE DOCKER_UPDATE_REBUILT_WEEKLY_FILE \
|
||||
WATCHDOG_CONTAINER_RESTART_LOG TUNING_MONITOR_LOG LIDARR_TRACKED_COUNT_FILE \
|
||||
LIDARR_RESCAN_DURATION_DB LIDARR_ART_MISS_CACHE LIDARR_DISCOVERY_HISTORY \
|
||||
SONARR_DISCOVERY_HISTORY RADARR_DISCOVERY_HISTORY SONARR_TRACKED_COUNT_FILE \
|
||||
CORRUPTION_SCAN_STATE_FILE CORRUPTION_SCAN_STRIKES_FILE RADARR_TRACKED_COUNT_FILE \
|
||||
TRANSCODE_DAILY_LOG BANDWIDTH_LOG ARR_CLEANUP_STATS ARR_RECOVERY_STATS \
|
||||
ARR_RECOVERY_FAILURE_COUNTS; do
|
||||
sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${DB_DIR}/|" "$CONF"
|
||||
done
|
||||
for v in AI_INDEX_DB AI_MEMORY_FILE AI_TOKEN_DB; do
|
||||
sed -i -E "s|^(\s*${v}=\")\\\$\{?DATA_DIR\}?/|\1\${AI_DATA_DIR}/|" "$CONF"
|
||||
done
|
||||
sed -i -E 's|^(\s*LIDARR_CACHE_FILE=")\$\{?DATA_DIR\}?/|\1${ARR_CACHE_BACKUP_DIR}/|' "$CONF"
|
||||
sed -i -E 's|^(\s*ZFS_REPORT_LOG=")\$\{?DATA_DIR\}?/|\1${LOG_ARCHIVE_DIR}/|' "$CONF"
|
||||
say "rewritten — backup kept beside it"
|
||||
else
|
||||
say "would rewrite STATE_DIR, PERSISTENT_CONF_CACHE and 25 file paths"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 2. Build the tree ─────────────────────────────────────────────────────────
|
||||
step "Step 2: directory tree"
|
||||
for d in "$DB_DIR" "$STATE_DIR" "$AI_DATA_DIR" "$ARR_CACHE_BACKUP_DIR" "$LOG_ARCHIVE_DIR"; do
|
||||
if [[ -d "$d" ]]; then say "exists ${d#$DATA_DIR/}"
|
||||
elif [[ "$DRY_RUN" == true ]]; then say "would create ${d#$DATA_DIR/}"
|
||||
else mkdir -p "$d" && say "created ${d#$DATA_DIR/}"
|
||||
fi
|
||||
done
|
||||
# The conf cache carries partner credentials and keeps its restrictive mode.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$CONF_CACHE_BACKUP_DIR" && chmod 700 "$CONF_CACHE_BACKUP_DIR"
|
||||
say "created cache/conf (0700)"
|
||||
fi
|
||||
|
||||
# ── 3. State files ────────────────────────────────────────────────────────────
|
||||
step "Step 3: State_Files → data/state"
|
||||
if [[ -d "$OLD_STATE" ]]; then
|
||||
shopt -s nullglob dotglob
|
||||
for f in "$OLD_STATE"/*; do move "$f" "$STATE_DIR"; done
|
||||
shopt -u nullglob dotglob
|
||||
if [[ "$DRY_RUN" == false && -d "$OLD_STATE" ]]; then
|
||||
rmdir "$OLD_STATE" 2>/dev/null && say "removed empty State_Files/" \
|
||||
|| say "State_Files/ not empty — left in place, inspect it"
|
||||
fi
|
||||
else
|
||||
say "no State_Files/ — nothing to do"
|
||||
fi
|
||||
|
||||
# ── 4. Sort the data root ─────────────────────────────────────────────────────
|
||||
step "Step 4: sort data/ into subfolders"
|
||||
classify() {
|
||||
local f="$1" base; base="$(basename "$f")"
|
||||
case "$base" in
|
||||
ai_*) move "$f" "$AI_DATA_DIR" ;;
|
||||
*_tracked_cache.json) move "$f" "$ARR_CACHE_BACKUP_DIR" ;;
|
||||
*.log) move "$f" "$LOG_ARCHIVE_DIR" ;;
|
||||
*.db|*.count|*.tsv|*.list|*.json) move "$f" "$DB_DIR" ;;
|
||||
*) say "leave $base — unclassified, left in data/" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Two passes, sidecars first. A SQLite database is three files, and the -wal holds committed
|
||||
# transactions that have not been checkpointed into the .db yet. Move the .db first and any
|
||||
# process that opens it during the gap sees a database with no write-ahead log, creates a fresh
|
||||
# one at the old path, and everything still in the old -wal is lost when it is moved over the
|
||||
# top. Sidecars ahead of their base closes that ordering: the worst case becomes a database
|
||||
# opened without its log still sitting beside it, which SQLite handles.
|
||||
shopt -s nullglob
|
||||
for f in "$DATA_DIR"/*-wal "$DATA_DIR"/*-shm; do
|
||||
[[ -d "$f" ]] && continue
|
||||
classify "$f"
|
||||
done
|
||||
for f in "$DATA_DIR"/*; do
|
||||
# Skip the destinations by name, not everything that happens to be a directory. ai_bugs/ and
|
||||
# ai_chats/ are directories that belong under ai/, and an earlier version of this skipped
|
||||
# every directory outright — which moved neither, silently, while the PHP layer had already
|
||||
# been repointed at the new location. They were empty at the time; that was luck, not design.
|
||||
case "$(basename "$f")" in
|
||||
db|state|ai|cache|logs) continue ;;
|
||||
*-wal|*-shm) continue ;;
|
||||
esac
|
||||
classify "$f"
|
||||
done
|
||||
shopt -u nullglob
|
||||
|
||||
# ── 5. Conf cache backup ──────────────────────────────────────────────────────
|
||||
step "Step 5: conf cache backup"
|
||||
if [[ -d "$OLD_CONFCACHE" ]]; then
|
||||
shopt -s nullglob dotglob
|
||||
for f in "$OLD_CONFCACHE"/*; do move "$f" "$CONF_CACHE_BACKUP_DIR"; done
|
||||
shopt -u nullglob dotglob
|
||||
[[ "$DRY_RUN" == false ]] && rmdir "$OLD_CONFCACHE" "$ROOT/.cache/vv" "$ROOT/.cache" 2>/dev/null
|
||||
else
|
||||
say "no $OLD_CONFCACHE — nothing to do"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────
|
||||
printf '\n━━━━━ SUMMARY ━━━━━\n'
|
||||
printf ' %-10s %s\n' "moved:" "$moved"
|
||||
printf ' %-10s %s\n' "skipped:" "$skipped"
|
||||
printf ' %-10s %s\n' "failed:" "$failed"
|
||||
[[ "$DRY_RUN" == true ]] && printf '\n DRY RUN — nothing was changed.\n'
|
||||
[[ "$failed" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -283,7 +283,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" =
|
||||
IDS=$(echo "$SEARCHES" | tr '{' '\n' | \
|
||||
grep '"isComplete":true' | grep '"searchText":' | \
|
||||
grep -o '"id":"[^"]*"' | sed 's/"id":"//;s/"//')
|
||||
COUNT=$(echo "$IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
COUNT=$(echo "$IDS" | grep -c . 2>/dev/null || true)
|
||||
COUNT="${COUNT//[^0-9]/}"; COUNT="${COUNT:-0}"
|
||||
|
||||
if [[ "$COUNT" -eq 0 ]]; then
|
||||
@@ -339,7 +339,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]] && [[ "$SLSKD_CONNECTED" =
|
||||
if [[ -z "$USERNAMES" ]]; then
|
||||
success "No transfer records found ✅"
|
||||
else
|
||||
USER_COUNT=$(echo "$USERNAMES" | grep -c . 2>/dev/null || echo 0)
|
||||
USER_COUNT=$(echo "$USERNAMES" | grep -c . 2>/dev/null || true)
|
||||
log "Found $USER_COUNT user(s) with transfer records"
|
||||
SUCCESS=0; SKIPPED=0; FAIL=0
|
||||
while IFS= read -r USER; do
|
||||
@@ -416,7 +416,7 @@ if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
else
|
||||
OLD_IMPORTS=$(find "$SLSKD_FAILED_IMPORTS_DIR" \
|
||||
-mindepth 1 -maxdepth 1 -mtime +"${DOWNLOADER_RETENTION_DAYS}")
|
||||
IMPORT_COUNT=$(echo "$OLD_IMPORTS" | grep -c . 2>/dev/null || echo 0)
|
||||
IMPORT_COUNT=$(echo "$OLD_IMPORTS" | grep -c . 2>/dev/null || true)
|
||||
IMPORT_COUNT="${IMPORT_COUNT//[^0-9]/}"; IMPORT_COUNT="${IMPORT_COUNT:-0}"
|
||||
|
||||
if [[ "$IMPORT_COUNT" -eq 0 ]]; then
|
||||
@@ -459,7 +459,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
if [[ -z "$COMPLETED_IDS" ]]; then
|
||||
success "No completed history found ✅"
|
||||
else
|
||||
HIST_TOTAL=$(echo "$COMPLETED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
HIST_TOTAL=$(echo "$COMPLETED_IDS" | grep -c . 2>/dev/null || true)
|
||||
log "Found $HIST_TOTAL completed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
@@ -507,7 +507,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
if [[ -z "$FAILED_IDS" ]]; then
|
||||
success "No failed history found ✅"
|
||||
else
|
||||
FAILED_TOTAL=$(echo "$FAILED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
FAILED_TOTAL=$(echo "$FAILED_IDS" | grep -c . 2>/dev/null || true)
|
||||
log "Found $FAILED_TOTAL failed history entries"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
@@ -557,7 +557,7 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
if [[ -z "$STALLED_IDS" ]]; then
|
||||
success "No stalled queue items found ✅"
|
||||
else
|
||||
QUEUE_TOTAL=$(echo "$STALLED_IDS" | grep -c . 2>/dev/null || echo 0)
|
||||
QUEUE_TOTAL=$(echo "$STALLED_IDS" | grep -c . 2>/dev/null || true)
|
||||
log "Found $QUEUE_TOTAL queue item(s) — checking status"
|
||||
DELETED=0; SKIPPED=0
|
||||
while IFS= read -r NZO_ID; do
|
||||
@@ -618,7 +618,7 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
-H "Cookie: $QBIT_COOKIE" 2>/dev/null)
|
||||
|
||||
NOW=$(date +%s)
|
||||
TORRENT_TOTAL=$(echo "$TORRENTS" | tr '}' '\n' | grep -c '"hash"' 2>/dev/null || echo 0)
|
||||
TORRENT_TOTAL=$(echo "$TORRENTS" | tr '}' '\n' | grep -c '"hash"' 2>/dev/null || true)
|
||||
log "Found $TORRENT_TOTAL torrent(s) — applying age/ratio filter"
|
||||
DELETED=0; SKIPPED=0
|
||||
|
||||
|
||||
@@ -314,7 +314,7 @@ FALLBACK_HOST2_WRITEBACK_TIER1=(
|
||||
Location: `$STATE_DIR/fallback_state.db` (survives reboots — boot device or appdata)
|
||||
|
||||
> In a shell where load_config.sh is not sourced, use the full path:
|
||||
> `/boot/config/plugins/varaverk/State_Files/fallback_state.db` (internal storage mode)
|
||||
> `/boot/config/plugins/varaverk/data/state/fallback_state.db` (internal storage mode)
|
||||
|
||||
```
|
||||
state=NORMAL # NORMAL | FALLBACK | NO_INTERNET | DARK
|
||||
|
||||
Executable
+351
@@ -0,0 +1,351 @@
|
||||
#!/bin/bash
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# PURPOSE
|
||||
# Put the containers this host has marked for fallback coverage onto the partner, so that the
|
||||
# partner can actually start them during an outage — and take them off again on request.
|
||||
#
|
||||
# OPERATIONAL MODEL
|
||||
# fallback.sh covers a host by running `docker start <name>` on the partner. It never creates
|
||||
# anything. So a name in FALLBACK_<me>_TIER* is a promise that only holds if the partner already
|
||||
# has that container built. Measured 2026-08-23: all 12 of HOST1's covered containers were absent
|
||||
# from HOST2, meaning every tier would have failed on the first real outage while the UI showed
|
||||
# coverage as configured. This script is what closes that gap.
|
||||
#
|
||||
# Push and remove are separate, deliberate actions, never a side effect of saving the tier list.
|
||||
# Editing coverage is a cheap config write; deploying a dozen containers onto another machine is
|
||||
# not, and the two should not share a button.
|
||||
#
|
||||
# DESIGN PRINCIPLES
|
||||
# Deployed, then verified STOPPED.
|
||||
# A container built here and left running on the partner would be a second live instance of
|
||||
# NextCloud, Gitea or PostgreSQL_Immich against the same data while this host is healthy.
|
||||
# That is the danger_rsync_live_database_appdata failure with worse odds. Every deploy is
|
||||
# followed by a stop and a re-inspect, and a container that will not stay stopped is an
|
||||
# error, not a warning.
|
||||
#
|
||||
# Remove takes the container AND its appdata.
|
||||
# Operator decision 2026-08-23: the button is explicit, so a removal should leave nothing
|
||||
# behind to reason about later. The risk it accepts is narrow and worth naming — if the
|
||||
# partner ever covered for us, ITS appdata is the newer copy and is what a handback rsyncs
|
||||
# home. The NORMAL-state gate below closes the live-failover window; what it cannot see is
|
||||
# a handback that partially failed and then returned to NORMAL, so the UI says so before
|
||||
# asking.
|
||||
#
|
||||
# Two guards on the deletion itself: only paths under /mnt/*/appdata* are ever touched, and
|
||||
# a bind of the appdata ROOT is refused outright — a container mounting /mnt/user/appdata
|
||||
# would otherwise turn one removal into wiping every application on the partner.
|
||||
#
|
||||
# Refuses to run unless fallback state is NORMAL.
|
||||
# Pushing or removing containers mid-outage edits the thing currently keeping services up.
|
||||
#
|
||||
# Coverage names are resolved to templates by <Name>, not by filename.
|
||||
# my-Foo.xml routinely holds a container called something else. Matching on the filename
|
||||
# silently pushes the wrong template, or nothing at all.
|
||||
#
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# Only in NORMAL state. FALLBACK_STATE_FILE is read before anything is pushed or removed, and
|
||||
# any other state refuses the action. A push during a live failover would deploy a second copy
|
||||
# of a container the partner is currently running on our behalf; a remove would delete the one
|
||||
# doing the covering.
|
||||
#
|
||||
# --status is exempt from that gate, because it only reports. Refusing to answer "what is
|
||||
# deployed over there" during a failover would withhold the information precisely when it is
|
||||
# most wanted.
|
||||
#
|
||||
# Every deploy is verified stopped, and a container that will not stay stopped is an error
|
||||
# rather than a warning — see DESIGN PRINCIPLES. A second live instance against the same data
|
||||
# is the failure this whole script exists inside.
|
||||
#
|
||||
# Push and remove are explicit modes with no default. Running the script with no flag does
|
||||
# nothing; neither action can be reached by accident, and neither is a side effect of editing
|
||||
# the tier list.
|
||||
#
|
||||
# --dry-run works in every mode and touches nothing on either host — no container is built,
|
||||
# started, stopped or removed, and no template is written or deleted.
|
||||
#
|
||||
# Remove deletes the container's appdata on the partner as well. That is deliberate and is the
|
||||
# most destructive thing here; the NORMAL-state gate above is what keeps it away from a
|
||||
# partner that is mid-handback.
|
||||
#
|
||||
# CONFIGURATION
|
||||
# master.conf
|
||||
# FALLBACK_<HOST>_TIER1..N the covered container names — what --push deploys and --status
|
||||
# reports on. This script reads that list; it never edits it.
|
||||
#
|
||||
# host*.conf
|
||||
# FALLBACK_STATE_FILE overrides where fallback.sh's state is read from. Defaults to
|
||||
# STATE_DIR/fallback_state.db. A missing file reads as NORMAL,
|
||||
# which is the correct default on a host where fallback has never
|
||||
# run.
|
||||
#
|
||||
# RUNTIME MODES
|
||||
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
|
||||
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
|
||||
# coverage_deploy.sh --status report, per covered container, whether it exists there
|
||||
# any mode supports --dry-run
|
||||
#
|
||||
# DEPENDS ON
|
||||
# Plugin/<platform>/Partnership/containers.sh deploy_container_from_xml(), GPU transform
|
||||
# FALLBACK_<me>_TIER1-4 the coverage list this acts on
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
||||
|
||||
SSH_TIMEOUT="${SSH_TIMEOUT:-15}"
|
||||
MODE=""
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--push) MODE="push" ;;
|
||||
--remove) MODE="remove" ;;
|
||||
--status) MODE="status" ;;
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$MODE" ]]; then
|
||||
error "No mode given — use --push, --remove or --status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "$REMOTE_ID" || "$REMOTE_SERVER_NAME" == "unknown" ]]; then
|
||||
error "No partner configured — nothing to push to"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Gate: only with fallback idle ─────────────────────────────────────────────────────────────
|
||||
# Read rather than assumed. A missing state file means fallback has never run, which is idle
|
||||
# enough; a file that says anything other than NORMAL means services are in motion right now.
|
||||
FALLBACK_STATE_FILE="${FALLBACK_STATE_FILE:-${STATE_DIR}/fallback_state.db}"
|
||||
_fb_state="NORMAL"
|
||||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||
_fb_state=$(grep -m1 '^state=' "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
_fb_state="${_fb_state:-NORMAL}"
|
||||
fi
|
||||
if [[ "$_fb_state" != "NORMAL" && "$MODE" != "status" ]]; then
|
||||
error "Fallback state is $_fb_state, not NORMAL — refusing to $MODE"
|
||||
error "Changing what the partner holds while a failover is live edits the thing keeping services up."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── The coverage list ─────────────────────────────────────────────────────────────────────────
|
||||
COVERED=()
|
||||
for _t in 1 2 3 4; do
|
||||
_var="FALLBACK_${MY_ID}_TIER${_t}[@]"
|
||||
for _c in "${!_var}"; do
|
||||
[[ -n "$_c" ]] && COVERED+=("$_c")
|
||||
done
|
||||
done
|
||||
|
||||
if [[ ${#COVERED[@]} -eq 0 ]]; then
|
||||
warn "No containers are covered in FALLBACK_${MY_ID}_TIER1-4 — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$ICON_FALLBACK Coverage: ${#COVERED[@]} container(s) for $REMOTE_SERVER_NAME to start during an outage"
|
||||
|
||||
resolve_remote_ip
|
||||
MIRROR="$REMOTE_SERVER_NAME"
|
||||
MIRROR_IP="$REMOTE_SERVER"
|
||||
_key_var="${MY_ID}_SSH_KEY"
|
||||
MIRROR_SSH_KEY="${!_key_var}"
|
||||
|
||||
if [[ ! -f "$MIRROR_SSH_KEY" ]]; then
|
||||
error "SSH key $MIRROR_SSH_KEY not found — cannot reach $MIRROR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── name -> template ──────────────────────────────────────────────────────────────────────────
|
||||
# Matched on the <Name> element. Filenames lie often enough that trusting them would push the
|
||||
# wrong container without saying so.
|
||||
xml_for_container() {
|
||||
local want="$1" f n
|
||||
for f in "$TEMPLATES_DIR"/*.xml; do
|
||||
[[ -f "$f" ]] || continue
|
||||
n=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$f")
|
||||
[[ "$n" == "$want" ]] && { echo "$f"; return 0; }
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
remote_has_container() {
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker inspect $(printf '%q' "$1") >/dev/null 2>&1" 2>/dev/null
|
||||
}
|
||||
|
||||
remote_state_of() {
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker inspect -f '{{.State.Status}}' $(printf '%q' "$1") 2>/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
OK=0; FAIL=0; SKIP=0
|
||||
|
||||
case "$MODE" in
|
||||
|
||||
status)
|
||||
# Written as a cache as well as printed. The assistant's fallback_state block cannot afford an
|
||||
# SSH round trip per container mid-question, so it reads this file and reports its AGE — a stale
|
||||
# answer stated as stale is useful, stated as current it is the exact failure this feature
|
||||
# exists to prevent.
|
||||
_present="" _missing=""
|
||||
for c in "${COVERED[@]}"; do
|
||||
if remote_has_container "$c"; then
|
||||
_st=$(remote_state_of "$c")
|
||||
printf ' %-28s on %s (%s)\n' "$c" "$MIRROR" "$_st"
|
||||
_present+="\"$c\":\"${_st:-unknown}\","
|
||||
OK=$((OK+1))
|
||||
else
|
||||
printf ' %-28s MISSING on %s — docker start would fail\n' "$c" "$MIRROR"
|
||||
_missing+="\"$c\","
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
done
|
||||
log "$ICON_FALLBACK Coverage present: $OK · missing: $FAIL"
|
||||
|
||||
mkdir -p "$VV_CACHE_ROOT/api" 2>/dev/null || mkdir -p /tmp/varaverk/api 2>/dev/null
|
||||
_cache="${VV_CACHE_ROOT:-/tmp/varaverk}/api/fallback_presence.json"
|
||||
# Written atomically — a half-written cache read mid-question would report containers as
|
||||
# missing that are merely unparsed.
|
||||
printf '{"present":{%s},"missing":[%s],"partner":"%s","checked":%s}\n' \
|
||||
"${_present%,}" "${_missing%,}" "$MIRROR" "$(date +%s)" > "$_cache.tmp" \
|
||||
&& mv -f "$_cache.tmp" "$_cache"
|
||||
|
||||
[[ "$FAIL" -gt 0 ]] && exit 2 || exit 0
|
||||
;;
|
||||
|
||||
push)
|
||||
# Networks first — a container whose network is absent is created and then cannot start,
|
||||
# which is the failure that read as "auth 0/8, arr 0/5" during onboarding.
|
||||
_nets=()
|
||||
for c in "${COVERED[@]}"; do
|
||||
x=$(xml_for_container "$c") || continue
|
||||
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$x" 2>/dev/null | head -1)
|
||||
net="${net//[[:space:]]/}"
|
||||
# br* is host hardware. wg* is a WireGuard-backed bridge whose meaning does NOT travel:
|
||||
# recreating it on the partner as a plain bridge yields a network that exists, starts its
|
||||
# containers, and routes their traffic OUTSIDE the tunnel. ChannelTube rides wg0 here.
|
||||
case "$net" in
|
||||
''|bridge|host|none|br[0-9]*) continue ;;
|
||||
wg[0-9]*)
|
||||
warn "$c uses $net — a WireGuard-backed network. NOT created on $MIRROR: a plain"
|
||||
warn " bridge of the same name would route its traffic outside the tunnel. Build the"
|
||||
warn " matching tunnel there first, or drop $c from coverage."
|
||||
continue ;;
|
||||
esac
|
||||
_seen=false
|
||||
for n in "${_nets[@]}"; do [[ "$n" == "$net" ]] && { _seen=true; break; }; done
|
||||
[[ "$_seen" == false ]] && _nets+=("$net")
|
||||
done
|
||||
for net in "${_nets[@]}"; do
|
||||
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" --format '{{.Driver}}' 2>/dev/null)
|
||||
if [[ "$driver" != "bridge" ]]; then
|
||||
warn "Network $net is '${driver:-absent}' here, not bridge — create it on $MIRROR by hand"
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would ensure network $net on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
||||
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null \
|
||||
&& log " network $net ready on $MIRROR" \
|
||||
|| warn " could not ensure network $net on $MIRROR"
|
||||
done
|
||||
|
||||
for c in "${COVERED[@]}"; do
|
||||
x=$(xml_for_container "$c") || {
|
||||
warn "$c — no template in $TEMPLATES_DIR names it; skipped"
|
||||
SKIP=$((SKIP+1)); continue
|
||||
}
|
||||
if ! deploy_container_from_xml "$x" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||
error "$c — deploy failed"
|
||||
FAIL=$((FAIL+1)); continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then OK=$((OK+1)); continue; fi
|
||||
|
||||
# Deployed containers must not run here. Stop, then re-inspect — a stop that did not take
|
||||
# is the one outcome that silently duplicates a live service against shared data.
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker stop $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null
|
||||
st=$(remote_state_of "$c")
|
||||
if [[ "$st" == "running" ]]; then
|
||||
error "$c is RUNNING on $MIRROR after deploy and would not stop — stop it there before continuing"
|
||||
FAIL=$((FAIL+1))
|
||||
else
|
||||
log " $c deployed and ${st:-stopped} on $MIRROR ✅"
|
||||
OK=$((OK+1))
|
||||
fi
|
||||
done
|
||||
log "$ICON_FALLBACK Push complete — deployed $OK · failed $FAIL · skipped $SKIP"
|
||||
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||
;;
|
||||
|
||||
remove)
|
||||
for c in "${COVERED[@]}"; do
|
||||
if ! remote_has_container "$c"; then
|
||||
log " $c not on $MIRROR — nothing to remove"
|
||||
SKIP=$((SKIP+1)); continue
|
||||
fi
|
||||
# Binds are read BEFORE the container goes — once it is removed there is nothing left to
|
||||
# enumerate, and a path list gathered afterwards would silently be empty.
|
||||
_binds=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' $(printf '%q' "$c") 2>/dev/null \
|
||||
| awk -F: '{print \$1}'" 2>/dev/null)
|
||||
|
||||
_wipe=()
|
||||
while IFS= read -r _p; do
|
||||
[[ -z "$_p" ]] && continue
|
||||
# Only appdata, and never an appdata root. /mnt/user/appdata as a bind would make one
|
||||
# container removal delete every application on the partner.
|
||||
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/.+ ]] || {
|
||||
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/?$ ]] && \
|
||||
warn " $c binds the appdata ROOT ($_p) — refusing to delete it"
|
||||
continue
|
||||
}
|
||||
_wipe+=("$_p")
|
||||
done <<< "$_binds"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop and remove $c on $MIRROR"
|
||||
for _p in "${_wipe[@]}"; do warn " DRY RUN — would delete appdata $_p on $MIRROR"; done
|
||||
OK=$((OK+1)); continue
|
||||
fi
|
||||
x=$(xml_for_container "$c") && xml_name=$(basename "$x") || xml_name=""
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker stop $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||
docker rm $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||
${xml_name:+rm -f ${TEMPLATES_DIR}/$(printf '%q' "$xml_name");} \
|
||||
! docker inspect $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null; then
|
||||
log " $c removed from $MIRROR ✅"
|
||||
for _p in "${_wipe[@]}"; do
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"rm -rf -- $(printf '%q' "$_p") && ! [ -e $(printf '%q' "$_p") ]" 2>/dev/null; then
|
||||
log " appdata deleted on $MIRROR: $_p"
|
||||
else
|
||||
warn " could not delete appdata on $MIRROR: $_p"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
done
|
||||
OK=$((OK+1))
|
||||
else
|
||||
error "$c — removal failed or it still exists on $MIRROR"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
done
|
||||
log "$ICON_FALLBACK Remove complete — removed $OK · failed $FAIL · skipped $SKIP"
|
||||
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||
;;
|
||||
esac
|
||||
+85
-4
@@ -314,7 +314,51 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt).
|
||||
|
||||
# ── Persistent dry-run ────────────────────────────────────────────────────────────────────────
|
||||
# Conf-driven, not argument-driven, deliberately. array_started.sh launches every entry as a bare
|
||||
# `bash script.sh &` with no arguments, so a --dry-run typed at a terminal survives exactly until
|
||||
# the next array start — and then the LIVE daemon comes up in its place, silently, which is the
|
||||
# one transition nobody would be watching for.
|
||||
#
|
||||
# Setting it here means the mode is a property of the install rather than of how the process
|
||||
# happened to be started: array start, the Fallback tab's button, and a hand-run all agree.
|
||||
#
|
||||
# OR, never override: --dry-run on the command line still wins over a conf that says false, so an
|
||||
# ad-hoc preview against a live install needs no conf edit.
|
||||
if [[ "${FALLBACK_DRY_RUN:-false}" == "true" ]]; then
|
||||
DRY_RUN=true
|
||||
fi
|
||||
|
||||
# ── Persistent log ────────────────────────────────────────────────────────────────────────────
|
||||
# array_started.sh launches every entry as a bare `bash script.sh &` with no redirection, so this
|
||||
# daemon's output has never been captured anywhere: /var/log/varaverk has a directory for every
|
||||
# other script family and none for Fallback. A month of dry-run observation would have persisted
|
||||
# nothing at all.
|
||||
#
|
||||
# /var/log is a 128 MB tmpfs on Unraid — RAM, and cleared on reboot — so the log goes to
|
||||
# LOG_ARCHIVE_DIR, which follows DATA_DIR onto real storage.
|
||||
#
|
||||
# Only when stdout is not a terminal. Run by hand, output still goes to the terminal exactly as
|
||||
# before; run by array_started or the Fallback tab's button, it lands in the file. A plain append
|
||||
# redirect rather than `tee` through process substitution: no extra child to outlive, and nothing
|
||||
# for the shutdown trap to race.
|
||||
FALLBACK_LOG="${LOG_ARCHIVE_DIR:-${DATA_DIR:-/tmp}/logs}/fallback.log"
|
||||
if [[ ! -t 1 ]]; then
|
||||
mkdir -p "$(dirname "$FALLBACK_LOG")" 2>/dev/null
|
||||
# One rotation, sized rather than line-counted — the whole point of this log is a long run,
|
||||
# and _orch_trim_log()'s 1000-line cap would discard weeks of it. Event-only output (no
|
||||
# --log) is a few lines per incident, so this holds years; --log fills it in about a fortnight
|
||||
# and then keeps the most recent fortnight plus the one before it.
|
||||
_fb_max=$(( ${FALLBACK_LOG_MAX_MB:-5} * 1048576 ))
|
||||
if [[ -f "$FALLBACK_LOG" ]] && (( $(stat -c %s "$FALLBACK_LOG" 2>/dev/null || echo 0) > _fb_max )); then
|
||||
mv -f "$FALLBACK_LOG" "${FALLBACK_LOG}.1" 2>/dev/null
|
||||
fi
|
||||
exec >> "$FALLBACK_LOG" 2>&1
|
||||
echo ""
|
||||
echo "═══ fallback.sh started $(date '+%Y-%m-%d %H:%M:%S') — dry_run=${DRY_RUN} pid=$$ ═══"
|
||||
fi
|
||||
# FALLBACK_ENABLED gate — exits cleanly when disabled.
|
||||
# Fail-closed: anything that isn't exactly "true" disables fallback. Matching only the
|
||||
# literal "false" would let a typo ("no", "0", "FALSE") hand this script DDNS authority
|
||||
# and cross-server container control on a toggle nobody meant to set.
|
||||
@@ -368,6 +412,22 @@ state_set() {
|
||||
}
|
||||
|
||||
state_init() {
|
||||
# A dry run must not leave the host believing it failed over. state_set() writes
|
||||
# unconditionally, and this file survives reboots and is what the real daemon — and the
|
||||
# Monitor and Fallback cards — read to decide what is happening. A --dry-run walk through
|
||||
# FAILOVER would have written state=FALLBACK, the tier flags and the strike counter into it
|
||||
# for real, and nothing would have put them back.
|
||||
#
|
||||
# Copied rather than merely redirected, so the preview still starts from the live state and
|
||||
# can advance through tiers exactly as a real run would. The copy lands in the RAM cache and
|
||||
# dies with the reboot.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
local live="$FALLBACK_STATE_FILE"
|
||||
FALLBACK_STATE_FILE="${VV_CACHE_ROOT:-/tmp/varaverk}/fallback_state.dryrun.$$"
|
||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||
if [[ -f "$live" ]]; then cp -f "$live" "$FALLBACK_STATE_FILE"; else : > "$FALLBACK_STATE_FILE"; fi
|
||||
warn "DRY RUN — state writes redirected to $FALLBACK_STATE_FILE (live state untouched)"
|
||||
fi
|
||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||||
@@ -759,8 +819,16 @@ run_handback() {
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||
|| warn "DRY RUN — would rsync: $job"
|
||||
# if/else, not A && B || C. In the shorthand a REAL run whose rsync exits
|
||||
# non-zero falls through to the || branch and logs "DRY RUN — would rsync",
|
||||
# so a failed Tier writeback reported itself as a preview and the real
|
||||
# failure went unsaid. The Tier 1 block below always had this right.
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||
|| error "Tier $tier writeback FAILED: $job"
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||||
@@ -904,6 +972,19 @@ echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
FALLBACK_RUNNING=true
|
||||
|
||||
# The dry-run state copy is per-PID and would otherwise accumulate one file per preview run.
|
||||
# EXIT as well as the signals, because a dry run is usually ended with Ctrl-C or --stop but can
|
||||
# also just fall out of the loop.
|
||||
dryrun_state_cleanup() {
|
||||
[[ "$DRY_RUN" == true && "$FALLBACK_STATE_FILE" == *".dryrun."* ]] && rm -f "$FALLBACK_STATE_FILE"
|
||||
return 0
|
||||
}
|
||||
# Must call _release_all_locks too. acquire_lock() registers its own EXIT trap, and bash keeps
|
||||
# exactly one per signal — a bare `trap ... EXIT` here silently replaced it and orphaned
|
||||
# fallback.lock, which is the precise failure the _LOCK_FILES registry in common.sh was built to
|
||||
# stop. The signal trap only needs `exit 0`; that fires EXIT, which does both jobs.
|
||||
trap 'dryrun_state_cleanup; _release_all_locks' EXIT
|
||||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
||||
SIGTERM SIGINT
|
||||
|
||||
@@ -1185,4 +1266,4 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
sleep "$FALLBACK_CHECK_INTERVAL" &
|
||||
wait $!
|
||||
|
||||
done
|
||||
done
|
||||
|
||||
@@ -91,13 +91,13 @@
|
||||
# FALLBACK_TEST_BLOCK_WAIT
|
||||
# Seconds to wait in Phase 3 for fallback.sh to detect the outage.
|
||||
# Must be > FALLBACK_CHECK_INTERVAL + buffer. At 30s interval: use ≥60s.
|
||||
# (default: 60)
|
||||
# (shipped default: 150)
|
||||
#
|
||||
# FALLBACK_TEST_HANDBACK_WAIT
|
||||
# Seconds to wait in Phase 6 for fallback.sh to complete handback.
|
||||
# Must cover: FALLBACK_HANDBACK_STRIKES × FALLBACK_CHECK_INTERVAL + rsync
|
||||
# duration + container start time. At 3 strikes × 30s + ~2min rsync +
|
||||
# ~1min container start: use ≥240s. (default: 300)
|
||||
# ~1min container start: use ≥240s. (shipped default: 360)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
@@ -118,6 +118,11 @@
|
||||
# fallback_test.sh --log
|
||||
# Verbose output on every check in every phase.
|
||||
#
|
||||
# fallback_test.sh --stop
|
||||
# Stop a running test. SIGTERM only — never SIGKILL, because only this script's EXIT
|
||||
# trap removes the iptables DROP rule it installed. Also sweeps a rule stranded by an
|
||||
# earlier SIGKILL or power cut.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -126,6 +131,58 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||||
#
|
||||
# SIGTERM ONLY, and deliberately no SIGKILL escalation — the opposite of fallback.sh --stop.
|
||||
# A running test holds an iptables DROP rule against the partner, and the only thing that removes
|
||||
# it is this script's own EXIT trap. SIGKILL does not run traps, so force-killing a test strands
|
||||
# the rule: the partner stays invisible, fallback.sh reads that as a permanent outage and holds
|
||||
# FALLBACK indefinitely. A test that will not die is a worse outcome than a test still running,
|
||||
# so this reports the stranded rule and the command to clear it rather than causing one.
|
||||
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||||
LOCKFILE="${LOCK_DIR}/fallback_test.lock"
|
||||
if [[ ! -f "$LOCKFILE" ]]; then
|
||||
log "No fallback_test.sh lock found — not running"
|
||||
exit 0
|
||||
fi
|
||||
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||||
target_pid="${lock_content%%:*}"
|
||||
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||||
warn "Stale lock — fallback_test.sh not running (PID ${target_pid:-unknown} gone) — clearing"
|
||||
rm -f "$LOCKFILE"
|
||||
# A stale lock is exactly the SIGKILL/power-cut case, so the rule may still be in place.
|
||||
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||
warn "Stranded iptables DROP rule found for $REMOTE_SERVER — removing"
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||
&& warn "Stranded rule removed — remote connectivity restored ✅" \
|
||||
|| error "Could not remove stranded rule — run: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
warn "Stopping fallback_test.sh (PID $target_pid) — SIGTERM so its trap clears the iptables rule..."
|
||||
kill -TERM "$target_pid" 2>/dev/null || true
|
||||
waited=0
|
||||
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 30 ]]; do
|
||||
sleep 1
|
||||
(( waited++ )) || true
|
||||
done
|
||||
if kill -0 "$target_pid" 2>/dev/null; then
|
||||
error "fallback_test.sh (PID $target_pid) did not exit within 30s"
|
||||
error "NOT force-killing — SIGKILL would strand the iptables DROP rule on $REMOTE_SERVER"
|
||||
error "Wait, or clear manually: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||
exit 1
|
||||
fi
|
||||
warn "Stopped: fallback_test.sh (PID $target_pid) ✅"
|
||||
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||
error "iptables DROP rule for $REMOTE_SERVER survived the stop — removing"
|
||||
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||
&& warn "Rule removed ✅" || error "Could not remove — run it by hand"
|
||||
else
|
||||
log "No iptables DROP rule remains for $REMOTE_SERVER ✅"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
@@ -325,6 +382,27 @@ else
|
||||
warn "No state file found — assuming NORMAL (first run)"
|
||||
fi
|
||||
|
||||
# fallback.sh must actually be RUNNING, not merely enabled
|
||||
#
|
||||
# Every phase after this one waits for the daemon to change state. FALLBACK_ENABLED=true says
|
||||
# it is allowed to run; it does not say array_started.sh launched it, or that it is still alive.
|
||||
# Without this the test passes pre-flight, drops a real iptables rule on the partner, waits
|
||||
# FALLBACK_TEST_BLOCK_WAIT for a transition nothing is there to make, and fails Phase 3 blaming
|
||||
# fallback detection. Only the EXIT trap gets connectivity back.
|
||||
#
|
||||
# In --dry-run nothing is blocked and nothing is waited on, so a dead daemon is worth saying but
|
||||
# not worth aborting for — the walkthrough still shows the operator the shape of the run.
|
||||
if pgrep -f "Fallback/fallback\.sh" >/dev/null 2>&1; then
|
||||
log "fallback.sh daemon is running"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "fallback.sh is NOT running — a real test would abort here"
|
||||
else
|
||||
error "fallback.sh is not running — nothing would detect the outage this test creates"
|
||||
error "Start it with array_started.sh, or run with --dry-run to walk the phases"
|
||||
phase_fail "Pre-flight"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Tier 1 containers configured
|
||||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||||
|
||||
Executable → Regular
+46
-10
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
SSH_TIMEOUT=15
|
||||
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
|
||||
TOTAL_MATCH=0
|
||||
TOTAL_MISMATCH=0
|
||||
TOTAL_MISSING=0
|
||||
TOTAL_UNVERIFIED=0
|
||||
SHARES_WITH_ISSUES=()
|
||||
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_UNVERIFIED=0
|
||||
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
# The path is interpolated into a remote shell command, so it must be escaped for
|
||||
# reuse as one word. A bare '$local_file' inside single quotes breaks on the first
|
||||
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
|
||||
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
|
||||
# MISMATCH against a file that is byte-identical on the remote.
|
||||
printf -v remote_q '%q' "$local_file"
|
||||
|
||||
# Existence and content are separate questions. Asking them together means a slow
|
||||
# checksum is indistinguishable from an absent file.
|
||||
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
"test -f $remote_q && echo yes" 2>/dev/null </dev/null)
|
||||
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
if [[ "$remote_exists" != "yes" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
|
||||
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
|
||||
# even though it was present and correct.
|
||||
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
|
||||
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
|
||||
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
|
||||
|
||||
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
# Present but unreadable within budget. Reporting this as a mismatch or a miss
|
||||
# would be a claim the run did not earn.
|
||||
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
|
||||
(( SHARE_UNVERIFIED++ ))
|
||||
(( TOTAL_UNVERIFIED++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
(( SHARE_MATCH++ ))
|
||||
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
done
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
|
||||
Executable → Regular
+18
-18
@@ -18,6 +18,22 @@
|
||||
# WebGUI slowdowns or timeouts under load.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Each run snapshots:
|
||||
# inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit.
|
||||
# Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT.
|
||||
# php-fpm: active workers vs PHP_MAX_CHILDREN limit.
|
||||
# Warns above PHP_FPM_WARN_PCT.
|
||||
#
|
||||
# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days):
|
||||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|
|
||||
# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||||
# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts
|
||||
# warnings over the week to show trend severity.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -36,22 +52,6 @@
|
||||
# signal — routine snapshots below the threshold produce nothing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Each run snapshots:
|
||||
# inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit.
|
||||
# Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT.
|
||||
# php-fpm: active workers vs PHP_MAX_CHILDREN limit.
|
||||
# Warns above PHP_FPM_WARN_PCT.
|
||||
#
|
||||
# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days):
|
||||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|
|
||||
# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||||
# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts
|
||||
# warnings over the week to show trend severity.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -209,7 +209,7 @@ fi
|
||||
# ==============================================================================================
|
||||
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"
|
||||
|
||||
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||||
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || true)
|
||||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
|
||||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"
|
||||
|
||||
@@ -259,4 +259,4 @@ echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_
|
||||
>> "$TUNING_MONITOR_LOG"
|
||||
|
||||
echo "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
|
||||
log "Entry: ${DATE}|${TIME}|${INOTIFY_USED}/${INOTIFY_LIMIT}(${INOTIFY_PCT}%,warn=${INOTIFY_WARN})|${PHPFPM_ACTIVE}/${PHPFPM_MAX}(${PHPFPM_PCT}%,warn=${PHPFPM_WARN})"
|
||||
log "Entry: ${DATE}|${TIME}|${INOTIFY_USED}/${INOTIFY_LIMIT}(${INOTIFY_PCT}%,warn=${INOTIFY_WARN})|${PHPFPM_ACTIVE}/${PHPFPM_MAX}(${PHPFPM_PCT}%,warn=${PHPFPM_WARN})"
|
||||
|
||||
Executable
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================== Uptime Report =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# The weekly read of what Tools/uptime_probe.sh has been recording every minute: anything down
|
||||
# right now, and anything that was not perfect over the last seven days. Runs in the Sunday
|
||||
# Morning Coffee Report.
|
||||
#
|
||||
# Silent on a clean week. A report that always says something is a report nobody reads, so this
|
||||
# prints nothing and notifies nothing when every domain was 100%.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# uptime_probe.php --report exits 1 when it has something to say and 0 when it does not, so the
|
||||
# decision to notify is the exit code rather than this script parsing the text it just printed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Silence is the normal output.
|
||||
# A report that always says something is a report nobody reads. A perfect week prints nothing
|
||||
# and notifies nothing, so anything that does appear in the Sunday report is worth the glance.
|
||||
#
|
||||
# The exit code is the decision, not the text.
|
||||
# uptime_probe.php --report exits 1 when it has something to say and 0 when it does not. This
|
||||
# script never parses the output it just printed to work out whether to notify — a report whose
|
||||
# wording changed would otherwise silently stop notifying.
|
||||
#
|
||||
# It reads; it never probes.
|
||||
# The measurements are already taken, once a minute, by Tools/uptime_probe.sh. Re-probing at
|
||||
# report time would describe Sunday morning rather than the week being reported on.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only. Reads the stored history and prints; records nothing, and cannot alter the data it
|
||||
# is reporting on.
|
||||
#
|
||||
# UPTIME_PROBE_ENABLED gates the whole run — with the probe off there is no history worth
|
||||
# reporting, and this says nothing rather than reporting an empty week as a perfect one.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# uptime_report.sh the weekly read. Silent when every domain was 100%.
|
||||
#
|
||||
# Called from COFFEE_REPORT_SCRIPTS; takes no arguments and has no other mode. For live figures
|
||||
# or a per-domain table, use Tools/uptime_probe.sh --status.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# UPTIME_PROBE_ENABLED nothing here runs when the probe is switched off
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
acquire_lock
|
||||
detect_hosts
|
||||
|
||||
if [[ "${UPTIME_PROBE_ENABLED:-true}" == "false" ]]; then
|
||||
log "$ICON_GEAR Uptime probe disabled — nothing to report"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
REPORT="$(php "$SCRIPT_DIR/../Plugin/unraid/Tools/uptime_probe.php" --report 2>/dev/null)"
|
||||
RC=$?
|
||||
|
||||
if [[ $RC -eq 0 || -z "$REPORT" ]]; then
|
||||
echo "$ICON_DONE All monitored domains at 100% this week ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "$REPORT"
|
||||
|
||||
DOWN_COUNT=$(grep -c "DOWN" <<< "$REPORT" || true)
|
||||
if [[ "$DOWN_COUNT" -gt 0 ]]; then
|
||||
notify "$DOWN_COUNT domain(s) currently unreachable on $(hostname)" "Uptime" "warning"
|
||||
else
|
||||
notify "Some domains had downtime this week on $(hostname)" "Uptime" "normal"
|
||||
fi
|
||||
exit 0
|
||||
Executable → Regular
+19
-19
@@ -14,24 +14,6 @@
|
||||
# into a single digest. Reads only — writes nothing, changes nothing.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Aggregator, Not Generator
|
||||
# This script reads state files that other scripts maintain. It never produces
|
||||
# health data itself — it only presents what is already there. Each source
|
||||
# script remains responsible for its own state; this script is the envelope.
|
||||
#
|
||||
# Profile-Driven Notification
|
||||
# The cron schedule never changes. The DIGEST_PROFILE in master.conf controls
|
||||
# when notifications actually send — switching from daily noise to weekly
|
||||
# summaries is a one-line conf change, not a cron edit.
|
||||
#
|
||||
# Read-Only, No Side Effects
|
||||
# Writes nothing, changes nothing, triggers nothing. Safe to run at any time
|
||||
# for a health snapshot without affecting any running service or state file.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -58,6 +40,24 @@
|
||||
# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Aggregator, Not Generator
|
||||
# This script reads state files that other scripts maintain. It never produces
|
||||
# health data itself — it only presents what is already there. Each source
|
||||
# script remains responsible for its own state; this script is the envelope.
|
||||
#
|
||||
# Profile-Driven Notification
|
||||
# The cron schedule never changes. The DIGEST_PROFILE in master.conf controls
|
||||
# when notifications actually send — switching from daily noise to weekly
|
||||
# summaries is a one-line conf change, not a cron edit.
|
||||
#
|
||||
# Read-Only, No Side Effects
|
||||
# Writes nothing, changes nothing, triggers nothing. Safe to run at any time
|
||||
# for a health snapshot without affecting any running service or state file.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -394,4 +394,4 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
elif [[ "$SHOULD_SEND" == true ]]; then
|
||||
notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
|
||||
echo "Digest sent"
|
||||
fi
|
||||
fi
|
||||
|
||||
Executable → Regular
+30
-21
@@ -15,6 +15,24 @@
|
||||
# comparison. In --dry-run mode, console only — nothing written to the log.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Five report sections (each skips gracefully if its data source is unavailable):
|
||||
#
|
||||
# ZFS pool health — status, state, errors per pool. Pools in
|
||||
# ZFS_REPORT_IGNORE_POOLS excluded from the report
|
||||
# (still fully monitored by unRAID — report-only exclusion).
|
||||
# ARC statistics — current ARC vs max, metadata pressure, hit rate.
|
||||
# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT, or if
|
||||
# ARC headroom (max - current) drops below ZFS_REPORT_ARC_FREE_WARN_GB.
|
||||
# Memory status — total, free, available RAM (informational only — see note below).
|
||||
# Warns if available < ZFS_REPORT_AVAIL_WARN_GB.
|
||||
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
|
||||
# Useful for spotting containers approaching watchdog limits.
|
||||
# Kernel pressure — vmstat snapshot (3 samples).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -42,24 +60,6 @@
|
||||
# reclaimable cache) is still checked separately as a true system-pressure signal.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Five report sections (each skips gracefully if its data source is unavailable):
|
||||
#
|
||||
# ZFS pool health — status, state, errors per pool. Pools in
|
||||
# ZFS_REPORT_IGNORE_POOLS excluded from the report
|
||||
# (still fully monitored by unRAID — report-only exclusion).
|
||||
# ARC statistics — current ARC vs max, metadata pressure, hit rate.
|
||||
# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT, or if
|
||||
# ARC headroom (max - current) drops below ZFS_REPORT_ARC_FREE_WARN_GB.
|
||||
# Memory status — total, free, available RAM (informational only — see note below).
|
||||
# Warns if available < ZFS_REPORT_AVAIL_WARN_GB.
|
||||
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
|
||||
# Useful for spotting containers approaching watchdog limits.
|
||||
# Kernel pressure — vmstat snapshot (3 samples).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -261,8 +261,17 @@ echo "━━━ $ICON_ZFS ARC Statistics ━━━"
|
||||
if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
warn "ZFS arcstats not available — skipping ARC section"
|
||||
else
|
||||
ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || \
|
||||
awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
# zfs_arc_max reads 0 when it has been left at the default, which is a value rather than a
|
||||
# failure — so the || fallback never fires for the case that actually needs it, exactly like a
|
||||
# grep -c that prints 0 and exits 1. Zero here would reach the ARC_PCT division below, and awk
|
||||
# treats division by zero as fatal: it prints nothing, ARC_PCT comes back empty, and the whole
|
||||
# ARC section reports blanks. c_max is the cap the kernel is really enforcing either way.
|
||||
ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || echo 0)
|
||||
ARC_MAX="${ARC_MAX//[^0-9]/}"
|
||||
if [[ "${ARC_MAX:-0}" -eq 0 ]]; then
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
|
||||
ARC_MAX="${ARC_MAX:-0}"
|
||||
fi
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
|
||||
@@ -381,4 +390,4 @@ fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
exit 0
|
||||
|
||||
+70
-3
@@ -1,7 +1,13 @@
|
||||
# Varaverk AI Integration — Design Notes
|
||||
|
||||
**Status: design only. Nothing below is built.** No Varaverk script calls Ollama, and no
|
||||
`AI_*` variable exists in any conf yet. Captured 2026-08-01 so the reasoning survives.
|
||||
**Status: retrieval is built; integration is not.** As of 2026-08-02 the `AI_*` and
|
||||
`HOST*_OLLAMA_*` variables exist in both confs and both templates, and `AI/` holds a working
|
||||
index and query path — see the RAG section at the end of this document and `AI/README-AI.md`.
|
||||
|
||||
Everything else below remains design only. **No Varaverk script consults AI.** Every
|
||||
`AI_ASSIST_*` toggle is false, `AI_CONF_WRITE_ENABLED` is false with an empty whitelist, and
|
||||
host resolution across the mesh is specified but not implemented. Originally captured
|
||||
2026-08-01 so the reasoning survives.
|
||||
|
||||
Ollama itself *is* installed, tuned and verified on HOST1 — `qwen2.5-coder:14b` for
|
||||
generation, `nomic-embed-text` for embeddings, 16k context, pinned to the RTX 3080. See
|
||||
@@ -420,7 +426,7 @@ to begin with.
|
||||
|
||||
Treat that as a deliberate boundary, not a happy accident:
|
||||
|
||||
- **index tracked files only** — never walk `Configurations/`, `State_Files/`, or `data/`
|
||||
- **index tracked files only** — never walk `Configurations/` or `data/`
|
||||
- a live conf value that the model genuinely needs should arrive through a *tool call* at
|
||||
query time, subject to the same redaction rules as everything else in the Security section,
|
||||
not be baked into a vector at index time
|
||||
@@ -490,3 +496,64 @@ new — the machinery already exists and was audited this session.
|
||||
possibly credentials pasted by a user.
|
||||
- Is a 7B worth it to buy context + parallelism headroom, or is 14B quality worth the
|
||||
serialisation? Defer until an actual problem is felt.
|
||||
|
||||
---
|
||||
|
||||
## RAG — built 2026-08-02
|
||||
|
||||
Retrieval is live. `AI/` holds the implementation; `AI/README-AI.md` documents it in full. What
|
||||
follows is only what changed relative to the plan recorded above.
|
||||
|
||||
**Corpus is larger than estimated.** ~2,950 chunks across ~180 files, not the 690 header chunks
|
||||
projected. Sub-chunking is why — see below.
|
||||
|
||||
**Named-paragraph sub-chunking was necessary, and was not in the plan.** Section-level chunks
|
||||
alone were too coarse. `rsync.sh` documents fourteen safeguards in one 2.8k-char
|
||||
`OPERATIONAL SAFEGUARDS` block; a query about one of them scored 0.558, below unrelated chunks,
|
||||
because the other thirteen dominated the vector. Splitting on the named-paragraph titles the
|
||||
header convention already uses took the same query to 0.718 and first place. The parent section
|
||||
name is carried onto each sub-chunk so routing still works.
|
||||
|
||||
**Two chunker bugs worth remembering.** The last section in a header (RUNTIME MODES in bash,
|
||||
DEPENDS ON in a page) ran to EOF and swept up every unrelated comment in the file —
|
||||
`scheduler.php` alone produced an 11k-char chunk of unrelated inline comments. And title
|
||||
detection must require the *next* line to be indented; without that, any wrapped prose line
|
||||
became a spurious chunk boundary mid-sentence.
|
||||
|
||||
**Section routing is a boost, not a filter.** Intent detection is a heuristic and must not be
|
||||
able to exclude the chunk holding the answer. `--section=` forces a hard filter when wanted.
|
||||
|
||||
**Vectors arrive pre-normalised.** `nomic-embed-text` returns L2-normalised vectors (measured
|
||||
norm 1.0000001), so cosine is a plain dot product. No normalising step, no magnitude cache.
|
||||
|
||||
**`node:sqlite` over a native module.** Still flagged experimental, chosen because it needs no
|
||||
native compilation on Unraid. Acceptable because the index is disposable — if a Node upgrade
|
||||
breaks it, rebuild takes minutes. PHP reads the same float32 blobs with `unpack('f*', $blob)`
|
||||
when the UI needs them.
|
||||
|
||||
**Retrieval quality, measured.** 9/10 top-3 hit rate on known-answer questions; the tenth had
|
||||
the answer at ranks 2 and 3, so 10/10 for answer-present-in-context at k=6. Full retrieval plus
|
||||
generation runs about 43s warm.
|
||||
|
||||
### The finding that validated the whole thing
|
||||
|
||||
First real end-to-end question asked which variable controls the mover's grace window. The model
|
||||
answered `MOVER_STOP_TIMEOUT`, "defaults to 30 seconds", citing `mover_stop.sh › CONFIGURATION`.
|
||||
Variable correct; the 30 was wrong — the real value is 300. The model was quoting the header
|
||||
verbatim. **The header was stale.**
|
||||
|
||||
A sweep for the same pattern found six stale `(default: N)` claims across the repo — all
|
||||
corrected in the same pass. This is the operating principle for the folder:
|
||||
|
||||
> Retrieval is exactly as accurate as the documentation it points at. When an answer looks
|
||||
> wrong, check the cited source before blaming the model.
|
||||
|
||||
It also means the index is a documentation-drift detector, not only a question-answering tool.
|
||||
|
||||
### Still deliberately not built
|
||||
|
||||
Nothing consults this. Every `AI_ASSIST_*` toggle is false, `AI_CONF_WRITE_ENABLED` is false
|
||||
with an empty key whitelist, and no watchdog, cleanup or fallback path calls it. Host resolution
|
||||
across the Tailscale mesh is specified above but not implemented — `ai_index.sh` and
|
||||
`ai_query.sh` currently require a local `HOST*_OLLAMA_URL` and fail with a clear message when it
|
||||
is empty, rather than silently probing the mesh.
|
||||
|
||||
@@ -266,16 +266,8 @@ echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no scripts launched"
|
||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# The configured list is the denominator — a script the conf names but that never launched is
|
||||
# skipped, not absent, and only shows up if something counts it.
|
||||
JOB_COUNT="${#ARRAY_START_SCRIPTS[@]}"
|
||||
orchestrator_summary "ARRAY START" "$START" "Array Start"
|
||||
exit $?
|
||||
@@ -190,18 +190,6 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Array Stop" "normal"
|
||||
else
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Stop" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
JOB_COUNT="$STEP"
|
||||
orchestrator_summary "ARRAY STOP" "$START" "Array Stop"
|
||||
exit $?
|
||||
|
||||
@@ -244,9 +244,28 @@ if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
||||
PARTNER_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
||||
|
||||
# A successful rsync proves the partner answered; a failed one is evidence it did not. But
|
||||
# rsync being switched off is neither — and it used to be read as "unseen", so the offline
|
||||
# counter climbed every 30 minutes toward the 30-day auto-offboard on a partnership whose
|
||||
# only fault was that RSYNC_ENABLED=false. That is how a deliberately paused sync ends up
|
||||
# dismantling the partnership it was paused for. With no rsync attempt there is nothing to
|
||||
# report, so the check runs without touching the counter either way.
|
||||
# Tier 2 counts as "switched off" here exactly as much as Tier 1 does. The guard below used
|
||||
# to test RSYNC_ENABLED alone, but it is CRITICAL_RSYNC_ENABLED that governs whether this
|
||||
# orchestrator attempts an rsync at all — so with Tier 1 open and Tier 2 closed, no transfer
|
||||
# was attempted, RSYNC_OK stayed false, and the run fell through to --remote-unseen and
|
||||
# incremented the counter every 30 minutes against a partner that was answering fine.
|
||||
#
|
||||
# Onboard Step 1d now leaves precisely that posture on purpose — Tier 1 open so provisioning
|
||||
# can run, every Tier 2 gate closed so nothing is scheduled. A freshly onboarded, perfectly
|
||||
# healthy partnership would have auto-offboarded itself 30 days later.
|
||||
if [[ "$RSYNC_OK" == true ]]; then
|
||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||
--check --remote-seen $PARTNER_DRY
|
||||
elif [[ "${RSYNC_ENABLED:-false}" != true || "${CRITICAL_RSYNC_ENABLED:-false}" != true ]]; then
|
||||
echo "Critical rsync gated off — partnership check runs, offline counter untouched"
|
||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||
--check $PARTNER_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||
--check --remote-unseen $PARTNER_DRY
|
||||
@@ -261,23 +280,7 @@ fi
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
# Standard ending, quiet mode — 30-min cadence, so a healthy cycle stays one line.
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
orchestrator_summary "CRITICAL SYNC" "$START" "Critical Sync" quiet
|
||||
exit $?
|
||||
@@ -354,11 +354,6 @@ WINDOW_END=$(date +%s)
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
@@ -382,16 +377,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Daily Maintenance" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
# Standard ending — derives skipped from SHARE_COUNT, so a run with rsync gated off reports
|
||||
# PARTIAL instead of "all complete".
|
||||
orchestrator_summary "DAILY MAINTENANCE" "$WINDOW_START" "Daily Maintenance"
|
||||
exit $?
|
||||
@@ -12,7 +12,8 @@
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.cache/vv/d/)
|
||||
# 1. conf_sync.sh — refresh partner conf cache in RAM (/tmp/varaverk/conf/),
|
||||
# both directions: pull theirs, push ours
|
||||
# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
||||
# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
||||
# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
||||
@@ -182,23 +183,27 @@ echo ""
|
||||
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Conf Pull ━━━
|
||||
# ━━━ Conf Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
||||
echo "━━━ $ICON_GEAR Conf Sync ━━━"
|
||||
|
||||
# Full sync, not --pull-only. The push half was written as an event-driven fast path for the
|
||||
# conf-save hook, but no such hook was ever built — so outside array start nothing pushed this
|
||||
# host's conf to its partners at all, and a partner's copy of our conf stayed at whatever it was
|
||||
# when we last rebooted. Pull alone kept our view of them fresh while their view of us decayed.
|
||||
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
||||
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
||||
warn "conf_sync.sh not found — skipping partner conf refresh"
|
||||
else
|
||||
_conf_args=("--pull-only")
|
||||
_conf_args=()
|
||||
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
|
||||
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
|
||||
echo "Partner conf cache refreshed ✅"
|
||||
JOB_PASS+=("conf_sync.sh --pull-only")
|
||||
JOB_PASS+=("conf_sync.sh")
|
||||
else
|
||||
warn "Partner conf pull failed — cache may be stale"
|
||||
JOB_FAIL+=("conf_sync.sh --pull-only")
|
||||
warn "Partner conf sync failed — cache may be stale"
|
||||
JOB_FAIL+=("conf_sync.sh")
|
||||
fi
|
||||
unset _conf_args
|
||||
fi
|
||||
@@ -352,20 +357,8 @@ if [[ "$SHOW_FULL" == true ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
if [[ "$SHOW_FULL" == true ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||
else
|
||||
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
||||
fi
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Intermediate Sync" "warning"
|
||||
fi
|
||||
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
|
||||
# anything skipped or failed expands to the full block on its own.
|
||||
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
|
||||
exit $?
|
||||
|
||||
@@ -307,18 +307,8 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Monthly Maintenance" "normal"
|
||||
else
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Monthly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped
|
||||
# step visible rather than absent.
|
||||
JOB_COUNT="$STEP"
|
||||
orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance"
|
||||
exit $?
|
||||
|
||||
@@ -98,6 +98,9 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
# Timed from here so the standard summary can report a real duration; this report had none.
|
||||
REPORT_START=$(date +%s)
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -184,6 +187,19 @@ echo "━━━━━ ☕ COFFEE REPORT SUMMARY ━━━━━"
|
||||
echo "🖥️ Host: $MY_ID"
|
||||
echo "⏱️ Done: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
|
||||
# Anything the Scheduler troubleshooter concluded was a Varaverk defect rather than a setting.
|
||||
# Counted here because a finding filed on a Tuesday and read on a Tuesday is a finding nobody
|
||||
# acts on — this report is the weekly moment the operator is actually looking. Count only: the
|
||||
# detail lives on the AI tab, and a report that reprints every open bug stops being skimmable.
|
||||
# Silent when there are none, and silent when AI is off, so a host without it reads the same as
|
||||
# it always has.
|
||||
if [[ "${AI_ENABLED:-false}" == true && -d "$DATA_DIR/ai_bugs" ]]; then
|
||||
_ai_bugs_open=$(grep -l '"open": true' "$DATA_DIR"/ai_bugs/*.json 2>/dev/null | wc -l)
|
||||
if [[ "${_ai_bugs_open:-0}" -gt 0 ]]; then
|
||||
echo "🐞 AI bugs: $_ai_bugs_open open — see the AI tab"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 ]]; then
|
||||
echo "✅ Passed: ${JOB_PASS[*]}"
|
||||
fi
|
||||
@@ -191,10 +207,9 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Sunday Morning Coffee Report" "warning"
|
||||
fi
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending. The configured section list is the denominator, so a report that quietly
|
||||
# stopped producing one of its sections reads as skipped rather than simply not appearing.
|
||||
JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}"
|
||||
[[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} ))
|
||||
orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report"
|
||||
exit $?
|
||||
|
||||
@@ -139,6 +139,10 @@ detect_hosts
|
||||
|
||||
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
||||
# from a healthy run. Fail loudly instead of silently doing no work.
|
||||
# This orchestrator never timed itself, so its summary could not report a duration. Set before
|
||||
# any work so the figure means the cycle, not the tail of it.
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
||||
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
||||
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
||||
@@ -223,15 +227,9 @@ done
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
|
||||
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||
else
|
||||
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
|
||||
if [[ "$DRY_RUN" != true ]]; then
|
||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Transcode Management" "warning"
|
||||
fi
|
||||
fi
|
||||
# Quiet by default — 7-min cadence. Anything failed or skipped expands on its own.
|
||||
JOB_COUNT="${#TRANSCODE_MANAGEMENT_SCRIPTS[@]}"
|
||||
orchestrator_summary "TRANSCODE CYCLE" "${CYCLE_START:-$(date +%s)}" "Transcode Management" quiet
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
|
||||
@@ -241,21 +241,15 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||
# ==============================================================================================
|
||||
# Per-script detail only when there is something to read; the standard block carries the rest.
|
||||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||||
fi
|
||||
|
||||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit 0
|
||||
# Quiet by default at a 15-min cadence. The configured script list is the denominator, so a
|
||||
# watchdog that silently stopped running one of its checks shows up as skipped.
|
||||
JOB_COUNT="${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||
orchestrator_summary "WATCHDOG CYCLE" "$CYCLE_START" "Watchdog Orchestrator" "$_mode"
|
||||
exit $?
|
||||
|
||||
@@ -402,18 +402,14 @@ WINDOW_END=$(date +%s)
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
# Per-unit detail first — the standard block that follows carries the verdict and the counts, not
|
||||
# the names, and knowing WHICH share failed is the whole point of reading a log.
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
@@ -422,19 +418,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Weekly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section
|
||||
# can no longer read as success — this is the run that printed "all complete — 0 shares synced".
|
||||
orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance"
|
||||
exit $?
|
||||
@@ -181,14 +181,36 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
|
||||
warn "DRY RUN — would remove HOST1 key from $MIRROR:/root/.ssh/authorized_keys"
|
||||
H1_DONE=true
|
||||
else
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
# `\|…|d`, not `|…|d`. sed only accepts a custom address delimiter when it is
|
||||
# introduced by a backslash; the bare form is a syntax error — "unknown command: `|'".
|
||||
# A delimiter other than / is still required, because the key blob is base64 and
|
||||
# routinely contains /.
|
||||
#
|
||||
# The error went to 2>/dev/null and `echo ok` ran anyway, so this reported
|
||||
# "key removed ✅" on every run while removing nothing, and a cancelled onboard left
|
||||
# HOST1's key live on the mirror. Report on what the remote actually did instead.
|
||||
_cancel_out=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||
"sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
|
||||
sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
|
||||
echo ok" 2>/dev/null | grep -q ok && {
|
||||
echo "HOST1 key removed from $MIRROR authorized_keys ✅"
|
||||
H1_DONE=true
|
||||
} || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
|
||||
"sed -i \"\\|${KEY_BLOB}|d\" /root/.ssh/authorized_keys || { echo sed-failed; exit 1; }
|
||||
chmod 600 /root/.ssh/authorized_keys 2>/dev/null
|
||||
sed -i \"/^${MIRROR_ID}_PHASE/d; /^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
|
||||
grep -qF '${KEY_BLOB}' /root/.ssh/authorized_keys 2>/dev/null && echo still-present || echo ok" 2>/dev/null)
|
||||
case "$_cancel_out" in
|
||||
*ok*)
|
||||
echo "HOST1 key removed from $MIRROR authorized_keys ✅"
|
||||
H1_DONE=true
|
||||
;;
|
||||
*still-present*)
|
||||
warn "HOST1 key still present in $MIRROR authorized_keys — remove it there manually"
|
||||
;;
|
||||
*sed-failed*)
|
||||
warn "Could not edit authorized_keys on $MIRROR — remove HOST1 key there manually"
|
||||
;;
|
||||
*)
|
||||
warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
|
||||
;;
|
||||
esac
|
||||
unset _cancel_out
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -228,7 +250,9 @@ if [[ "$DIRECTION" == "h2" || "$DIRECTION" == "both" ]]; then
|
||||
warn "DRY RUN — would remove $MIRROR_SHORT key from $AUTH_KEYS"
|
||||
H2_DONE=true
|
||||
else
|
||||
sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && {
|
||||
# chmod after: sed -i rewrites via a temp file, which lands under the umask and can
|
||||
# leave the file 0666. sshd StrictModes then silently refuses every key in it.
|
||||
sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && chmod 600 "$AUTH_KEYS" && {
|
||||
echo "$MIRROR key removed from HOST1 authorized_keys ✅"
|
||||
H2_DONE=true
|
||||
} || warn "Failed to remove $MIRROR key from HOST1 authorized_keys"
|
||||
|
||||
@@ -351,16 +351,53 @@ EOF
|
||||
) 200>"${file}.lock"
|
||||
}
|
||||
|
||||
# Deliver the partnership state to the partner, at the path the partner actually reads, under
|
||||
# both names its UI looks for.
|
||||
#
|
||||
# This used to be one scp to "root@ip:$local_file" — the LOCAL absolute path reused verbatim on
|
||||
# the remote. That only works while both hosts install to the same place. HOST1 is on flash at
|
||||
# /boot/config/plugins/varaverk and HOST2 is in appdata mode at /mnt/user/appdata/Varaverk, so
|
||||
# the copy went to a directory HOST2 does not read and, more often, does not have — and the
|
||||
# failure surfaced as "will propagate on next sync", which nothing does.
|
||||
#
|
||||
# Both names, because the page resolves one file per node: partnership_<that node's hostname>.db.
|
||||
# The mirror needs partnership_<mirror>.db for its own card and partnership_<owner>.db for the
|
||||
# owner's. Nothing on the mirror writes the first one during an owner-driven onboard, which is
|
||||
# why a mirror that was fully onboarded still rendered as having no partnership at all.
|
||||
# The content is symmetric — state/owner/mirror/onboarded — so one file serves as both.
|
||||
push_state_to_remote() {
|
||||
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push state file to remote"
|
||||
return 0
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
||||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
||||
echo "State file pushed to remote ✅" || \
|
||||
warn "Could not push state file to remote — will propagate on next sync"
|
||||
|
||||
# Resolve where the partner keeps its state, from the partner. varaverk.cfg names its
|
||||
# SCRIPTS_DIR; absent it, the flash default is the right guess for a stock install.
|
||||
local remote_sd remote_state_dir
|
||||
remote_sd=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \
|
||||
'grep -oP "(?<=SCRIPTS_DIR=\")[^\"]+" /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' \
|
||||
2>/dev/null | tr -d '\r')
|
||||
[[ -z "$remote_sd" ]] && remote_sd="/boot/config/plugins/varaverk"
|
||||
remote_state_dir="${remote_sd}/data/state"
|
||||
|
||||
local rc=0
|
||||
local name
|
||||
for name in "partnership_${REMOTE_SERVER_NAME}.db" "partnership_${LOCAL_SERVER_NAME}.db"; do
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \
|
||||
"mkdir -p '$remote_state_dir' && cat > '${remote_state_dir}/${name}'" \
|
||||
< "$local_file" 2>/dev/null || rc=1
|
||||
done
|
||||
|
||||
if [[ $rc -eq 0 ]]; then
|
||||
echo "State file pushed to $REMOTE_SERVER_NAME:${remote_state_dir} ✅"
|
||||
return 0
|
||||
fi
|
||||
warn "Could not push state to $REMOTE_SERVER_NAME:${remote_state_dir} — it will keep showing"
|
||||
warn " no partnership until this succeeds. Nothing retries this on a schedule."
|
||||
return 1
|
||||
}
|
||||
|
||||
read_remote_state() {
|
||||
@@ -467,6 +504,7 @@ do_ssh_key_revocation() {
|
||||
"grep -v '${our_comment}' /root/.ssh/authorized_keys \
|
||||
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
|
||||
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
|
||||
&& chmod 600 /root/.ssh/authorized_keys \
|
||||
&& echo removed" 2>/dev/null | grep -q removed; then
|
||||
echo "Our pubkey revoked from $REMOTE_SERVER_NAME ✅"
|
||||
SSH_REVOKE_REMOTE_OK=true
|
||||
@@ -490,9 +528,16 @@ do_ssh_key_revocation() {
|
||||
SSH_REVOKE_LOCAL_OK=true
|
||||
elif [[ -f /root/.ssh/authorized_keys ]]; then
|
||||
if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then
|
||||
# chmod after the mv, every time. `>` creates the temp file under the shell's umask
|
||||
# and `mv` keeps the NEW file's mode, so this rewrite left authorized_keys 0666 on a
|
||||
# filesystem that permits it. sshd's StrictModes then refuses every key in it without
|
||||
# saying so to the client — the key is present, byte-correct, and inert, and the next
|
||||
# onboard's SSH step fails with nothing in any Varaverk log to explain it. Only
|
||||
# /var/log/syslog knows: "Authentication refused: bad ownership or modes".
|
||||
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
|
||||
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
|
||||
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then
|
||||
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys && \
|
||||
chmod 600 /root/.ssh/authorized_keys; then
|
||||
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
|
||||
SSH_REVOKE_LOCAL_OK=true
|
||||
else
|
||||
@@ -532,6 +577,10 @@ gather_partner_fallback_containers() {
|
||||
|
||||
# Start this server's own parked containers after partnership ends.
|
||||
start_own_stack() {
|
||||
# Returns non-zero if any container failed. It used to return whatever the loop's last
|
||||
# docker start happened to produce, so a caller checking it learned nothing — and the
|
||||
# offboard summary just printed "Step 6 — Own stack: started" either way.
|
||||
local _rc=0
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Restart Own Stack ━━━"
|
||||
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
|
||||
@@ -548,14 +597,20 @@ start_own_stack() {
|
||||
echo "$container started ✅"
|
||||
else
|
||||
warn "$container failed to start — check manually"
|
||||
_rc=1
|
||||
fi
|
||||
done
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
# Remove partnership containers on this server + their appdata bind-mount paths.
|
||||
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
|
||||
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
|
||||
cleanup_partner_containers() {
|
||||
# Returns non-zero if any container or appdata path could not be removed. Previously the
|
||||
# exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅"
|
||||
# was printed over a container that failed to remove.
|
||||
local _rc=0
|
||||
declare -a containers=()
|
||||
gather_partner_fallback_containers containers
|
||||
|
||||
@@ -586,8 +641,12 @@ cleanup_partner_containers() {
|
||||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
|
||||
_PM_TRAP_STOPPED+=("$container")
|
||||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
|
||||
echo "$container removed ✅" || warn "$container rm failed"
|
||||
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
|
||||
echo "$container removed ✅"
|
||||
else
|
||||
warn "$container rm failed"
|
||||
_rc=1
|
||||
fi
|
||||
else
|
||||
log "$container not found — skipping"
|
||||
fi
|
||||
@@ -600,8 +659,14 @@ cleanup_partner_containers() {
|
||||
warn " DRY RUN — would rm -rf $path"
|
||||
continue
|
||||
fi
|
||||
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
||||
if rm -rf "$path"; then
|
||||
echo " Appdata removed: $path ✅"
|
||||
else
|
||||
warn " Failed to remove: $path"
|
||||
_rc=1
|
||||
fi
|
||||
done <<< "$all_appdata_paths"
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
|
||||
@@ -609,6 +674,10 @@ cleanup_partner_containers() {
|
||||
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
|
||||
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
|
||||
cleanup_owner_containers_on_mirror() {
|
||||
# Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly
|
||||
# rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the
|
||||
# containers on it are gone.
|
||||
local _rc=0
|
||||
local mirror_ip="$1"
|
||||
local owner_short
|
||||
owner_short=$(derive_short_name "$OWNER")
|
||||
@@ -639,24 +708,31 @@ cleanup_owner_containers_on_mirror() {
|
||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
||||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||||
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||
"docker stop '$container' >/dev/null 2>&1
|
||||
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||||
grep -q removed && \
|
||||
echo "$container removed from $MIRROR ✅" || \
|
||||
grep -q removed; then
|
||||
echo "$container removed from $MIRROR ✅"
|
||||
else
|
||||
warn "Failed to remove $container from $MIRROR"
|
||||
_rc=1
|
||||
fi
|
||||
|
||||
# Delete appdata on remote after container removal
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
|
||||
echo " Appdata removed on $MIRROR: $path ✅"
|
||||
else
|
||||
warn " Failed to remove appdata on $MIRROR: $path"
|
||||
_rc=1
|
||||
fi
|
||||
done <<< "$appdata_paths"
|
||||
done <<< "$container_list"
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
# SSH to mirror — start mirror's own parked containers.
|
||||
@@ -678,18 +754,23 @@ start_mirror_own_stack() {
|
||||
fi
|
||||
|
||||
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
|
||||
local _rc=0
|
||||
for container in "${mirror_own[@]}"; do
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
||||
grep -q started && \
|
||||
echo "$container started on $MIRROR ✅" || \
|
||||
grep -q started; then
|
||||
echo "$container started on $MIRROR ✅"
|
||||
else
|
||||
warn "$container failed to start on $MIRROR — check manually"
|
||||
_rc=1
|
||||
fi
|
||||
done
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
# Create the mirror's Emby admin account on the owner's deployed Emby.
|
||||
@@ -866,28 +947,70 @@ check_both_healthy() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Returns non-zero when the mirror did NOT leave with current state, so the caller can say so.
|
||||
# The whole point of this step is the guarantee in the summary — "mirror leaves with current
|
||||
# Critical-Data" — and it used to print that unconditionally at the end of the function. An
|
||||
# offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs"
|
||||
# immediately followed by "Final sync complete — mirror has current state ✅", and the summary
|
||||
# scored Step 2 as a pass. The mirror left with whatever it happened to have.
|
||||
do_final_sync() {
|
||||
log "Running final critical sync..."
|
||||
local _synced=0 _failed=0
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||||
# Tier 1 only, deliberately. rsync.sh honours RSYNC_ENABLED and nothing else — the Tier 2
|
||||
# gates belong to the orchestrators, so a direct call like this one runs whether or not
|
||||
# CRITICAL_RSYNC_ENABLED is set. Checking Tier 2 here would refuse a final sync that works
|
||||
# perfectly well, and the offboard's whole reason for existing is to get current auth data
|
||||
# onto the mirror before the keys go.
|
||||
#
|
||||
# What was wrong was the accounting below it: a share was counted synced whenever Tier 1
|
||||
# happened to be on, with rsync.sh's exit status ignored entirely. A transfer that failed
|
||||
# on a full disk, an offline share or a refused connection still reported "Final sync ✅
|
||||
# — mirror has current state", which is the one claim in this script somebody acts on.
|
||||
local _gate_ok=true
|
||||
if [[ "${RSYNC_ENABLED:-true}" != true ]]; then
|
||||
warn "RSYNC_ENABLED=false — Tier 1 stops every rsync, no share can be sent"
|
||||
_gate_ok=false
|
||||
fi
|
||||
|
||||
if [[ "$_gate_ok" != true ]]; then
|
||||
_failed=1
|
||||
elif [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||||
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||||
[[ -z "$_share" ]] && continue
|
||||
local _path="${_share%%|*}"
|
||||
local _profile="${_share##*|}"
|
||||
# Count on what rsync.sh returned, now that a closed gate can no longer reach here.
|
||||
if [[ "$_path" == "$_profile" ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
||||
--profile="$_profile" --log
|
||||
fi
|
||||
if [[ $? -eq 0 ]]; then (( _synced++ )) || true; else (( _failed++ )) || true; fi
|
||||
done
|
||||
else
|
||||
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
||||
_failed=1
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"
|
||||
# ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution",
|
||||
# so this line aborted Step 2 of every offboard --dry-run with a shell error instead of
|
||||
# printing. A count of an unset array is already 0, which is the only default needed.
|
||||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)"
|
||||
return 0
|
||||
fi
|
||||
warn "Final sync complete — mirror has current state ✅"
|
||||
|
||||
if [[ "$_failed" -gt 0 ]]; then
|
||||
warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had"
|
||||
[[ "${RSYNC_ENABLED:-true}" != true ]] && \
|
||||
warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent"
|
||||
[[ "${CRITICAL_RSYNC_ENABLED:-true}" != true ]] && \
|
||||
warn " CRITICAL_RSYNC_ENABLED=false — the Tier 2 gate stopped it"
|
||||
return 1
|
||||
fi
|
||||
echo "Final sync complete — $_synced share(s) sent, mirror has current state ✅"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Safe master.conf modification with error handling — appends the key if not already present,
|
||||
|
||||
@@ -13,28 +13,32 @@
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# OWNER PATH (10 steps)
|
||||
# OWNER PATH (11 steps)
|
||||
# Step 1: Stop rsync — halt any running sync before state changes
|
||||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||||
# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 4: Disarm sync gates — RSYNC/CRITICAL_RSYNC/CONF_SYNC/ARR_SYNC=false in master.conf,
|
||||
# the exact inverse of onboard Step 9c
|
||||
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
||||
# Step 6: Restart own stack — bring up owner's own parked containers
|
||||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||||
# Step 9: Revocation — Emby admin, SSH keys
|
||||
# Step 9: Emby revocation — remove the mirror's Emby admin while Emby is still reachable
|
||||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||
# Tailscale — grace window then device removal (after state written)
|
||||
# Tailscale — grace deadline recorded, device removed after it expires
|
||||
# Step 11: SSH revocation — keys, both directions. Genuinely last: it is the step that
|
||||
# removes the access every step above depends on
|
||||
#
|
||||
# MIRROR PATH (8 steps)
|
||||
# Step 1: Stop rsync — halt any running sync
|
||||
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
||||
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
||||
# Step 4: Fallback cleanup — remove fallback coverage containers
|
||||
# Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 5: Disarm sync gates — same four gates as the owner path
|
||||
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
||||
# Step 7: Restart own stack — bring up own parked containers
|
||||
# Step 8: SSH revocation — revoke keys both directions, write state, signal owner
|
||||
# Step 8: Finalise — write INACTIVE, clear phase flags, signal owner, THEN revoke
|
||||
# keys. Revocation is last because the signal needs the key
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
@@ -102,9 +106,12 @@
|
||||
# The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on —
|
||||
# stale access cannot survive the offboard.
|
||||
#
|
||||
# Tailscale Grace Window
|
||||
# Tailscale Grace Window Is a Deadline, Not a Sleep
|
||||
# Device removal happens after state is written, not before, so the final state push
|
||||
# cannot be cut off by removing its own transport.
|
||||
# cannot be cut off by removing its own transport. The grace period itself is recorded to
|
||||
# STATE_DIR/tailscale_removal_due.db and the offboard returns. It used to sleep
|
||||
# PARTNERSHIP_GRACE_HOURS inline — six hours by default — holding the lock and its job record
|
||||
# open the whole time, reporting "running", and blocking any re-onboard behind it.
|
||||
#
|
||||
# Dry Run Support
|
||||
# --dry-run walks the full sequence reporting each step without executing any.
|
||||
@@ -154,6 +161,19 @@ source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
||||
|
||||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||||
REASON="manual"
|
||||
STEP_DISABLE_RSYNC_OK=true # both paths report it; only the mirror path re-initialised it
|
||||
TAILSCALE_REMOVED=false # set only when remove_tailscale_device actually succeeds
|
||||
|
||||
# Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from
|
||||
# MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded.
|
||||
# An offboard that failed to remove a single container still reported a clean teardown.
|
||||
STEP_LOCAL_CLEANUP_OK=true
|
||||
STEP_OWN_STACK_OK=true
|
||||
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
|
||||
STEP_MIRROR_STACK_OK=true # or "skipped"
|
||||
STEP_STATE_WRITE_OK=true
|
||||
STEP_STATE_PUSH_OK=false # INACTIVE actually delivered to the mirror, or "skipped"
|
||||
STEP_SETUP_PUSH_OK=false # cleared phase flags delivered to the mirror, or "skipped"
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
@@ -214,6 +234,57 @@ echo " Reason: $REASON"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: disarm every sync gate a completed onboard armed ─────────────────────────────────
|
||||
#
|
||||
# The exact inverse of partnership_onboard.sh Step 9c, which sets RSYNC_ENABLED,
|
||||
# CONF_SYNC_ENABLED and ARR_SYNC_ENABLED to true on a completed onboard. Offboard used to turn
|
||||
# off CRITICAL_RSYNC_ENABLED and nothing else, which left the far more consequential Tier 1
|
||||
# RSYNC_ENABLED switched on with no partner to sync to.
|
||||
#
|
||||
# That asymmetry read as safe and was not. Turning off one Tier 2 gate looks like "sync is off"
|
||||
# in the summary, while INTERMEDIATE, DAILY, WEEKLY and FALLBACK rsync all stayed live behind an
|
||||
# open Tier 1 — and CONF_SYNC_ENABLED kept a 4-hourly job reaching for a partner that had just
|
||||
# been removed, failing and notifying each time.
|
||||
#
|
||||
# CRITICAL_RSYNC_ENABLED stays in the list. It is a Tier 2 gate and closing Tier 1 already stops
|
||||
# it, but leaving it true would misreport the state to anyone reading the conf rather than the
|
||||
# tier logic.
|
||||
#
|
||||
# FALLBACK_ENABLED is deliberately NOT here. Onboard does not arm it, so offboard has no business
|
||||
# disarming it — it is the operator's switch, and the summary says so rather than moving it.
|
||||
# ==============================================================================================
|
||||
_VV_SYNC_GATES=(RSYNC_ENABLED CRITICAL_RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED)
|
||||
|
||||
# ── Stop any running rsync, and say honestly whether it worked ────────────────────────────────
|
||||
#
|
||||
# Both offboard paths called "$SCRIPTS_ROOT/Rsync/rsync_stop.sh". That file has never existed —
|
||||
# rsync_stop.sh lives in System_Essentials/. With stderr sent to /dev/null the "No such file"
|
||||
# went unseen, and the line below it printed "Rsync stopped ✅" unconditionally, so every
|
||||
# offboard ever run reported stopping an rsync it had not touched. On the owner path the failed
|
||||
# exit also set STEP_STOP_OK=false, which is why the summary said ❌ two lines under a ✅.
|
||||
#
|
||||
# One helper, one path, and the outcome is the return value.
|
||||
stop_rsync_now() {
|
||||
local script="$SCRIPTS_ROOT/System_Essentials/rsync_stop.sh"
|
||||
if [[ ! -f "$script" ]]; then
|
||||
warn "rsync_stop.sh not found at $script"
|
||||
return 1
|
||||
fi
|
||||
bash "$script" --rsync-only
|
||||
}
|
||||
|
||||
_disarm_sync_gates() {
|
||||
local gate rc=0 conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||||
for gate in "${_VV_SYNC_GATES[@]}"; do
|
||||
# set_conf_bool, not update_master_conf: the latter rewrites the whole line and would
|
||||
# strip the trailing comment that explains what each tier gates. Same helper onboard
|
||||
# arms with, so arming and disarming are one operation in two directions.
|
||||
set_conf_bool "$gate" "false" "$conf" || rc=1
|
||||
done
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
||||
#
|
||||
@@ -305,8 +376,12 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
echo "Rsync stopped ✅"
|
||||
if stop_rsync_now; then
|
||||
echo "Rsync stopped ✅"
|
||||
else
|
||||
warn "Could not stop rsync — a transfer may still be running into $OWNER"
|
||||
STEP_STOP_RSYNC_OK=false
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
fi
|
||||
@@ -335,16 +410,14 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
|
||||
cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false
|
||||
|
||||
# ── Step 5: Disable critical sync ─────────────────────────────────────────────────────────
|
||||
# ── Step 5: Disarm the sync gates ─────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━"
|
||||
echo "━━━ $ICON_GEAR Step 5/8 — Disarm Sync Gates ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅" || \
|
||||
{ warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; }
|
||||
_disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}"
|
||||
fi
|
||||
|
||||
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
||||
@@ -357,19 +430,31 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
||||
|
||||
do_ssh_key_revocation "${OWNER_IP:-}"
|
||||
|
||||
# State first, keys last — same ordering the owner path needed. Revocation used to run here,
|
||||
# before the push below, so the mirror destroyed the key and then tried to tell the owner it
|
||||
# had left using that key. The owner never heard, and the notify promised it would "finalise
|
||||
# on next check" — a check that now had no way in.
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
echo "Local state: INACTIVE ✅"
|
||||
add_to_blocklist "$OWNER" "$REASON"
|
||||
|
||||
# Inverse of onboard's write_onboard_phase. MIRROR_ID is this host on this path, and the
|
||||
# flags are named for the mirror on both sides, so the same names clear here.
|
||||
_setup_db="$(platform_setup_db_path)"
|
||||
for _flag in "${MIRROR_ID}_PHASE1_DONE" "${MIRROR_ID}_PHASE2_DONE" "${MIRROR_ID}_KEY_READY"; do
|
||||
clear_state_var "$_setup_db" "$_flag"
|
||||
done
|
||||
echo "Onboard phase flags cleared ✅"
|
||||
unset _setup_db _flag
|
||||
else
|
||||
warn "DRY RUN — would write INACTIVE state and blocklist $OWNER"
|
||||
warn "DRY RUN — would write INACTIVE state, clear phase flags and blocklist $OWNER"
|
||||
fi
|
||||
|
||||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||||
platform_push_setup_state 2>/dev/null || warn "Could not push cleared setup state to $OWNER"
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
||||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||||
"Partnership" "normal"
|
||||
@@ -377,6 +462,9 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||||
fi
|
||||
|
||||
# Last, for the reason above: everything before it needs the key.
|
||||
do_ssh_key_revocation "${OWNER_IP:-}"
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
@@ -402,7 +490,7 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
||||
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
||||
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
||||
echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")"
|
||||
echo " Step 5 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
|
||||
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
||||
echo " Step 7 — Own stack: started"
|
||||
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
||||
@@ -441,8 +529,14 @@ echo ""
|
||||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
|
||||
echo "Rsync stopped ✅"
|
||||
if stop_rsync_now; then
|
||||
echo "Rsync stopped ✅"
|
||||
else
|
||||
warn "Could not stop rsync — a transfer may still be running into $MIRROR"
|
||||
warn " Everything below changes state while data is still moving, which is the one"
|
||||
warn " ordering this step exists to prevent. Check: ps -ef | grep rsync"
|
||||
STEP_STOP_OK=false
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
fi
|
||||
@@ -475,25 +569,24 @@ else
|
||||
(( WEBUI_FAILURES++ ))
|
||||
fi
|
||||
|
||||
# ── Step 4: Disable critical sync ─────────────────────────────────────────────────────────────
|
||||
# ── Step 4: Disarm the sync gates ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━"
|
||||
echo "━━━ $ICON_GEAR Step 4/10 — Disarm Sync Gates ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||||
_disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}"
|
||||
fi
|
||||
|
||||
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
||||
|
||||
cleanup_partner_containers
|
||||
cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false
|
||||
|
||||
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
start_own_stack || STEP_OWN_STACK_OK=false
|
||||
|
||||
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
@@ -501,64 +594,160 @@ echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false
|
||||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false
|
||||
else
|
||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
||||
STEP_REMOTE_CLEANUP_OK=skipped
|
||||
fi
|
||||
|
||||
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
||||
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false
|
||||
else
|
||||
STEP_MIRROR_STACK_OK=skipped
|
||||
fi
|
||||
|
||||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
||||
# ── Step 9: Emby revocation ───────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
||||
echo "━━━ $ICON_SHIELD Step 9/11 — Emby Revocation ━━━"
|
||||
|
||||
# Emby admin — before SSH key revocation while Emby still reachable
|
||||
# Before SSH key revocation, while Emby is still reachable
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||
|
||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||||
# State is written after container cleanup and SSH revocation so that:
|
||||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
||||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
||||
# Ahead of SSH revocation, which is now Step 11.
|
||||
#
|
||||
# "State Written Both Ends" is the guarantee this step exists for — neither side left believing
|
||||
# the partnership is active. It could not deliver it: revocation used to run here in Step 9 and
|
||||
# then this push authenticated with the key it had just destroyed, so every offboard ended
|
||||
# "Could not push state file to remote — will propagate on next sync". There is no next sync;
|
||||
# the gates are closed and the keys are gone. The mirror was left reading ACTIVE for ever.
|
||||
#
|
||||
# The original ordering note said state must follow revocation so a crash between steps 5–9
|
||||
# re-runs from scratch rather than early-exiting on INACTIVE. Writing it one step later than the
|
||||
# cleanup preserves that — the cleanup is still done before any state is recorded — while
|
||||
# putting the push back inside the window where it can actually reach the mirror.
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
||||
echo "━━━ $ICON_GEAR Step 10/11 — Write State ━━━"
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
echo "Local state: INACTIVE ✅"
|
||||
# Checked, because this is the record every other host and every later --check reads. A
|
||||
# failed write here leaves both sides believing the partnership is still active while the
|
||||
# summary says INACTIVE — the one line in the teardown that must not be assumed.
|
||||
if write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then
|
||||
echo "Local state: INACTIVE ✅"
|
||||
else
|
||||
error "Failed to write local state file — $MIRROR may still look ACTIVE here"
|
||||
STEP_STATE_WRITE_OK=false
|
||||
fi
|
||||
add_to_blocklist "$MIRROR" "$REASON"
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
|
||||
# The inverse of onboard's write_onboard_phase, which had none. Without this a completed
|
||||
# offboard leaves the state file reading INACTIVE beside HOST*_PHASE1_DONE / _PHASE2_DONE
|
||||
# still set — and the setup checklist and partnership card read the flags, not the state
|
||||
# file, so a torn-down partnership went on presenting itself as fully onboarded.
|
||||
#
|
||||
# KEY_READY goes too: it means "a key is generated and waiting to be installed", which stops
|
||||
# being true the moment Step 11 revokes both sides.
|
||||
_setup_db="$(platform_setup_db_path)"
|
||||
for _flag in "${MIRROR_ID}_PHASE1_DONE" "${MIRROR_ID}_PHASE2_DONE" "${MIRROR_ID}_KEY_READY"; do
|
||||
clear_state_var "$_setup_db" "$_flag"
|
||||
done
|
||||
echo "Onboard phase flags cleared ✅"
|
||||
unset _setup_db _flag
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
# Two separate pushes, and the summary used to report only the second. A run where the
|
||||
# setup.db push failed and the state-file push succeeded printed "Could not push state
|
||||
# file to remote" in the body and "Pushed to mirror: ✅" in the summary — describing
|
||||
# different files with the same words. They are tracked apart now, because they fail
|
||||
# apart: the mirror can be told the partnership is INACTIVE while keeping the phase
|
||||
# flags that make its own UI still claim a finished onboard.
|
||||
#
|
||||
# Pushed after the flags are cleared, so the mirror receives the cleared file rather than
|
||||
# the version that still claimed a finished onboard.
|
||||
if platform_push_setup_state 2>/dev/null; then
|
||||
STEP_SETUP_PUSH_OK=true
|
||||
else
|
||||
warn "Could not push cleared phase flags to $MIRROR — its wizard and partnership"
|
||||
warn " card will keep showing a completed onboard until it is reinstalled or pulled"
|
||||
fi
|
||||
if push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||
STEP_STATE_PUSH_OK=true
|
||||
else
|
||||
warn "Could not push INACTIVE to $MIRROR — it will keep reading ACTIVE until told otherwise"
|
||||
fi
|
||||
else
|
||||
STEP_STATE_PUSH_OK=skipped
|
||||
STEP_SETUP_PUSH_OK=skipped
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||||
STEP_STATE_PUSH_OK=true
|
||||
STEP_SETUP_PUSH_OK=true
|
||||
fi
|
||||
|
||||
# ── Step 11: SSH key revocation ───────────────────────────────────────────────────────────────
|
||||
# Genuinely last. Every step above needs working remote access — this is the one that takes it
|
||||
# away, so it goes after the final thing that has to reach the mirror.
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 11/11 — SSH Key Revocation ━━━"
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||||
# The grace period is recorded as a deadline, not slept through.
|
||||
#
|
||||
# This used to `sleep $((PARTNERSHIP_GRACE_HOURS * 3600))` inline — six hours by default —
|
||||
# holding the offboard's lock and its job record open the whole time, showing "running" to
|
||||
# every status reader, and blocking any re-onboard behind the lock. Worse, the sleep ran
|
||||
# even when removal was going to be a no-op: TAILSCALE_API_KEY and TAILSCALE_TAILNET are
|
||||
# both empty here, so the six hours bought nothing at all.
|
||||
#
|
||||
# The offboard's own work is finished by this point. Writing the deadline lets the teardown
|
||||
# complete now and leaves the removal to whoever reads the file — and makes the wait
|
||||
# visible and cancellable instead of buried in a sleeping process.
|
||||
_grace_h="${PARTNERSHIP_GRACE_HOURS:-6}"
|
||||
if [[ "$_grace_h" -gt 0 ]] && [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
_due=$(( $(date +%s) + _grace_h * 3600 ))
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||||
sleep "$grace_seconds"
|
||||
trap - SIGTERM SIGINT
|
||||
printf 'host=%s\ndue=%s\ndue_human=%s\nreason=%s\n' \
|
||||
"$MIRROR" "$_due" "$(date -d "@$_due" '+%Y-%m-%d %H:%M:%S')" "$REASON" \
|
||||
> "${STATE_DIR}/tailscale_removal_due.db"
|
||||
fi
|
||||
warn "Grace period: $MIRROR stays on the tailnet until $(date -d "@$_due" '+%Y-%m-%d %H:%M') — recorded, not slept"
|
||||
# No CLI entry point removes it yet, and there is deliberately no invented one here:
|
||||
# remove_tailscale_device() is a partnership_manager.sh function with no --mode of its
|
||||
# own, and it no-ops without credentials regardless. Say what is true.
|
||||
if [[ -z "${TAILSCALE_API_KEY:-}" || -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||||
warn "Automatic removal is not possible — TAILSCALE_API_KEY/TAILSCALE_TAILNET are unset; remove it in the Tailscale admin console"
|
||||
else
|
||||
warn "Removal after that is not yet automated — remove it in the Tailscale admin console"
|
||||
fi
|
||||
TAILSCALE_REMOVED=deferred
|
||||
unset _grace_h _due
|
||||
else
|
||||
# No grace configured, or the mirror is already unreachable — remove now.
|
||||
#
|
||||
# Outcome recorded, not assumed. remove_tailscale_device returns 1 when TAILSCALE_API_KEY
|
||||
# or TAILSCALE_TAILNET is unset — it warns "skipping Tailscale removal" and the summary
|
||||
# went on to report "removed ✅" anyway, so an offboard that left the device on the
|
||||
# tailnet said it had taken it off. Neither key is configured here, so that was every run.
|
||||
if remove_tailscale_device "$MIRROR"; then
|
||||
TAILSCALE_REMOVED=true
|
||||
fi
|
||||
fi
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
|
||||
# Backup handover notification
|
||||
@@ -597,24 +786,60 @@ _revoke_status() {
|
||||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
||||
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
||||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Step 4 — Disable sync: ✅"
|
||||
echo " Step 5 — Local cleanup: ✅"
|
||||
echo " Step 6 — Own stack: started"
|
||||
echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
||||
echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
||||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
||||
echo " Step 10 — State: INACTIVE ✅"
|
||||
echo " Step 4 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
|
||||
echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")"
|
||||
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
|
||||
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
|
||||
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
|
||||
echo " Step 9 — Emby revoked: $(_ok "${STEP_EMBY_OK:-true}")"
|
||||
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
|
||||
echo " Step 10 — INACTIVE pushed: $( [[ "$STEP_STATE_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_STATE_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still reads ACTIVE"; } )"
|
||||
echo " Step 10 — Phase flags pushed: $( [[ "$STEP_SETUP_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_SETUP_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still shows a finished onboard"; } )"
|
||||
echo " Step 11 — Keys revoked: $(_revoke_status)"
|
||||
echo ""
|
||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||||
echo " Tailscale: $MIRROR removed ✅"
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
||||
case "$TAILSCALE_REMOVED" in
|
||||
true) echo " Tailscale: $MIRROR removed ✅" ;;
|
||||
deferred) echo " Tailscale: $MIRROR kept until the grace period expires — see ${STATE_DIR}/tailscale_removal_due.db" ;;
|
||||
*) echo " Tailscale: $MIRROR NOT removed ⚠ — still on the tailnet (needs TAILSCALE_API_KEY + TAILSCALE_TAILNET)" ;;
|
||||
esac
|
||||
fi
|
||||
# Named because it is the one partnership switch neither onboard nor offboard moves, so it
|
||||
# survives an offboard still true and there is nothing else that would ever mention it.
|
||||
[[ "${FALLBACK_ENABLED:-false}" == true ]] && \
|
||||
echo " FALLBACK_ENABLED is still true — nothing left to fail over to, turn it off yourself"
|
||||
echo ""
|
||||
echo " $MIRROR leaves with:"
|
||||
echo " ✓ Current auth config (final sync)"
|
||||
echo " ✓ Auth WebUIs → localhost"
|
||||
# Each line reads the step that produced it. "✓ Current auth config (final sync)" was a
|
||||
# literal, so an offboard whose Step 2 had just warned "Final sync did NOT complete" still
|
||||
# closed by telling the operator the mirror held current auth data. That is the one claim here
|
||||
# somebody might act on — it is the difference between a partner that can stand alone and one
|
||||
# carrying a stale copy of the auth stack.
|
||||
if [[ "$STEP_SYNC_OK" == true ]]; then
|
||||
echo " ✓ Current auth config (final sync)"
|
||||
else
|
||||
echo " ✗ Auth config NOT synced — it keeps whatever it already had (see Step 2)"
|
||||
fi
|
||||
if [[ "${WEBUI_FAILURES:-0}" -eq 0 ]]; then
|
||||
echo " ✓ Auth WebUIs → localhost"
|
||||
else
|
||||
echo " ✗ ${WEBUI_FAILURES} auth WebUI(s) still point at $LOCAL_SERVER_NAME — fix by hand there"
|
||||
fi
|
||||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||||
|
||||
# The verdict is derived, never asserted. "DONE — clean separation complete ✅" printed
|
||||
# unconditionally, under a summary that had already shown two failed steps.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$STEP_STOP_OK" == true && "$STEP_SYNC_OK" == true \
|
||||
&& "$STEP_STATE_PUSH_OK" != false && "$STEP_SETUP_PUSH_OK" != false \
|
||||
&& "${WEBUI_FAILURES:-0}" -eq 0 ]]; then
|
||||
echo "$ICON_DONE DONE — clean separation complete ✅"
|
||||
else
|
||||
warn "$ICON_DONE Offboard finished with unresolved steps — the partnership is ended, but"
|
||||
warn " the ❌ lines above did not happen. Re-read them before re-onboarding."
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
|
||||
@@ -22,6 +22,12 @@
|
||||
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
||||
# Step 1b: Docker network — ensure varaverk docker network exists on mirror
|
||||
# Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent)
|
||||
# Step 1c2:Permit mirror — drop the mirror from the partnership blocklist a previous
|
||||
# offboard wrote, or Step 1e's rsync is refused
|
||||
# Step 1d: Sync gates — Tier 1 + CONF_SYNC + ARR_SYNC on, every Tier 2 rsync gate
|
||||
# off. Here, not at the end, because Step 1e is an rsync
|
||||
# Step 1e: Auth appdata — rsync PARTNERSHIP_PROVISION_SHARES to the mirror BEFORE the
|
||||
# containers that read it exist. The only rsync an onboard does
|
||||
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
||||
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
||||
# Mariadb/Redis health-checked before Authelia deploys
|
||||
@@ -32,10 +38,19 @@
|
||||
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
||||
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
||||
# Step 9b: Webhook setup — register download webhook in arrs on both servers
|
||||
# Step 9c: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed)
|
||||
# prevents arrs treating every file as missing after bootstrap
|
||||
# Step 9d: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
||||
# Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
||||
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
||||
# Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it
|
||||
# would discover are actually deployed there
|
||||
# Step 12: Container grouping — reproduce this host's folder layout on the mirror (Arrs
|
||||
# Stack, Networking, Databases…) for the containers deployed
|
||||
# there; only unfiled or PARTNERSHIP_FALLBACK_ONLY entries go
|
||||
# to "<OwnerShort>-Fallback". Icon resolved here and passed
|
||||
# over: the mirror has no Emby key
|
||||
#
|
||||
# Phase 3 (media seed) is NOT part of the above. Onboard ends at Phase 2 — connected, running,
|
||||
# and saying so. Seeding is a separate operator-triggered phase; see --phase3-only. With
|
||||
# MEDIA_SEED_ENABLED=false there is no Phase 3 and the model is two phases.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
@@ -150,16 +165,27 @@
|
||||
# Partnership/partnership_onboard.sh --skip-webhook-setup
|
||||
# Skip webhook registration in arrs (Step 9b)
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-media-seed
|
||||
# Skip initial media share rsync to mirror (Step 9c)
|
||||
# Use when mirror already has files or you want to seed manually
|
||||
# Partnership/partnership_onboard.sh --phase3-only
|
||||
# OWNER: media seed only. Refuses unless Phase 2 is done and MEDIA_SEED_ENABLED is true.
|
||||
# Sets the seeding gate posture (Tier 1 open, every Tier 2 gate closed) and dispatches
|
||||
# Rsync/media_seed.sh detached. Returns in seconds; the seed runs for weeks.
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-appdata-provision
|
||||
# Skip the pre-container auth appdata rsync (Step 1e)
|
||||
# The mirror's auth containers then start against whatever is already there
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --skip-webhook-listener
|
||||
# Skip starting webhook listener on mirror (Step 9d)
|
||||
# Skip starting webhook listener on mirror (Step 9e)
|
||||
# Listener will start automatically on next array restart
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --no-arm
|
||||
# Leave every sync gate as it is (Step 1d). Step 1e then has no Tier 1 and cannot provision.
|
||||
# Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose
|
||||
# shares are not populated yet.
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --phase1-only
|
||||
# OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
|
||||
# OWNER only: SSH key exchange, conf push, docker network, partner conf cache.
|
||||
# Safe to run before HOST2 has Varaverk — all of it needs docker and SSH, not the plugin.
|
||||
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
||||
#
|
||||
# Partnership/partnership_onboard.sh --phase2-only
|
||||
@@ -184,10 +210,13 @@ SKIP_ARR_STACK=false
|
||||
SKIP_SERVICES_STACK=false
|
||||
SKIP_ARR_SYNC=false
|
||||
SKIP_WEBHOOK_SETUP=false
|
||||
SKIP_MEDIA_SEED=false
|
||||
SKIP_WEBHOOK_LISTENER=false
|
||||
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
|
||||
PHASE1_ONLY=false # OWNER: SSH + conf push only (mirror not yet installed)
|
||||
# MIRROR: SSH key install only, no owner notification
|
||||
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
||||
PHASE3_ONLY=false # OWNER: media seed only — a separate decision, see the PHASE 3 block
|
||||
SKIP_ARM=false # leave the sync gates as they are — see Step 1d
|
||||
SKIP_APPDATA_PROVISION=false # skip the pre-container auth appdata sync — see Step 1e
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
@@ -199,10 +228,12 @@ for arg in "$@"; do
|
||||
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
||||
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
||||
--skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;;
|
||||
--skip-media-seed) SKIP_MEDIA_SEED=true ;;
|
||||
--skip-appdata-provision) SKIP_APPDATA_PROVISION=true ;;
|
||||
--skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;;
|
||||
--phase1-only) PHASE1_ONLY=true ;;
|
||||
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
||||
--phase3-only) PHASE3_ONLY=true; SKIP_SSH=true ;;
|
||||
--no-arm) SKIP_ARM=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
@@ -238,9 +269,15 @@ write_onboard_phase() {
|
||||
local state_file="$(platform_setup_db_path)"
|
||||
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
||||
set_state_var "$state_file" "$key" "true"
|
||||
platform_push_setup_state
|
||||
# The push now reports failure, so it is warned about rather than returned. This function's
|
||||
# contract is "the phase is recorded here" — the flag is on local disk either way, and an
|
||||
# unreachable partner must not make a completed phase look like it did not happen.
|
||||
platform_push_setup_state \
|
||||
|| warn "Phase flag ${key} written locally but not delivered to the partner"
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
@@ -297,14 +334,47 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
||||
echo ""
|
||||
|
||||
# Tested before attempted, the same guard the owner's phase 1 has had all along.
|
||||
#
|
||||
# The key step is a TERMINAL step by design — ssh_setup.sh runs ssh-copy-id, which prompts for
|
||||
# the owner's root password on a first install. Once the operator has done that in a terminal,
|
||||
# pressing ▶ Onboard ran the whole thing again: another ssh-copy-id, this time from the WebGUI
|
||||
# with no TTY to answer the prompt, which fails and aborts the run at Step 1 — so the button
|
||||
# whose entire job is Step 2 could never reach it.
|
||||
#
|
||||
# Working SSH is the actual precondition, not "have we run the setup script". If it already
|
||||
# works there is nothing to install, whichever route installed it.
|
||||
OWNER_IP_PRE=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
||||
if [[ "$SKIP_SSH" == true ]]; then
|
||||
warn "Skipping SSH setup (--skip-ssh)"
|
||||
elif [[ -n "$OWNER_IP_PRE" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$OWNER_IP_PRE" exit 0 2>/dev/null; then
|
||||
echo "SSH to $OWNER already works ✅ — key already installed, skipping setup"
|
||||
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||
echo "SSH key ready ✅"
|
||||
else
|
||||
error "SSH key setup failed"
|
||||
error "Install the key from a terminal on this host — ssh-copy-id needs $OWNER's password,"
|
||||
error "and a WebGUI button has no way to answer that prompt:"
|
||||
error " bash $SCRIPTS_ROOT/Partnership/partnership_onboard.sh --phase1-only"
|
||||
exit 1
|
||||
fi
|
||||
unset OWNER_IP_PRE
|
||||
|
||||
# Stop after the key when asked. ssh_setup.sh runs ssh-copy-id, which prompts for the
|
||||
# owner's root password on a first install — answerable in a terminal, never from the
|
||||
# WebGUI button, which is why the mirror's panel sends the operator to a terminal for
|
||||
# exactly this step and nothing more. The flag was parsed but only ever honoured on the
|
||||
# owner path, so a mirror asked for phase 1 silently ran the whole thing.
|
||||
if [[ "$PHASE1_ONLY" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MIRROR PHASE 1 COMPLETE ━━━━━"
|
||||
echo " SSH key: ready"
|
||||
echo " Next: press ▶ Onboard on the Partnership tab to notify $OWNER"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
||||
@@ -318,17 +388,77 @@ if [[ "$AM_MIRROR" == true ]]; then
|
||||
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
||||
PHASE2_TRIGGERED=true
|
||||
elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
|
||||
2>/dev/null | grep -q triggered; then
|
||||
echo "Phase 2 triggered on $OWNER ✅"
|
||||
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
|
||||
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2 via run_job.sh"
|
||||
PHASE2_TRIGGERED=true
|
||||
else
|
||||
warn "Could not auto-trigger Phase 2 on $OWNER"
|
||||
# Launched through run_job.sh, the same path cron and api/run.php use, so Phase 2
|
||||
# gets a stat file, a job log and a Scheduler entry on the owner. It used to be a
|
||||
# bare `nohup … > /tmp/vv_phase2_onboard.log`, which ran fine and left the owner's
|
||||
# entire half of onboarding invisible to its own WebGUI — no job record, nothing
|
||||
# under /var/log/varaverk, nothing for the operator to look at when asking why
|
||||
# pressing Onboard here appeared to do nothing.
|
||||
#
|
||||
# setsid, not bare nohup: the job must lead its own process group so api/stop.php
|
||||
# can signal the whole tree. api/run.php carries the same note for the same reason.
|
||||
#
|
||||
# Reported triggered only after the stat file proves run_job.sh actually started.
|
||||
# The old `& echo triggered` printed unconditionally — it would have claimed success
|
||||
# for a path that does not exist on the owner, which is exactly the failure mode a
|
||||
# mirror in a different storage mode hits.
|
||||
# Its own timeout, not SSH_TIMEOUT: the remote waits for the runner to prove itself,
|
||||
# and 15s would cut that short and report a healthy launch as a failure.
|
||||
_phase2_out=$(timeout 40 ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||
"bash -s -- $(printf '%q' "$OWNER_SCRIPTS_DIR")" 2>/dev/null <<'PHASE2_TRIGGER'
|
||||
sd="$1"
|
||||
runner="$sd/Plugin/unraid/run_job.sh"
|
||||
script="$sd/Partnership/partnership_onboard.sh"
|
||||
stat_file="/var/log/varaverk/Partnership/partnership_onboard.json"
|
||||
|
||||
[ -f "$runner" ] || { echo "missing-runner:$runner"; exit 1; }
|
||||
[ -f "$script" ] || { echo "missing-script:$script"; exit 1; }
|
||||
|
||||
# Absolute, not "newer than the file we saw a moment ago". A previous run's stat file rewritten
|
||||
# inside the same second would compare equal and read as a failed launch.
|
||||
t0=$(date +%s)
|
||||
|
||||
setsid nohup bash "$runner" "Partnership/partnership_onboard.sh" "$script" \
|
||||
--manual --phase2-only >/dev/null 2>&1 </dev/null &
|
||||
|
||||
# run_job.sh writes the stat file before it execs the script, so a stat file dated at or after
|
||||
# the moment we launched is proof the runner is alive — rather than proof the ssh call returned.
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
|
||||
sleep 1
|
||||
[ -f "$stat_file" ] || continue
|
||||
now=$(date -r "$stat_file" +%s 2>/dev/null || echo 0)
|
||||
[ "$now" -ge "$t0" ] && { echo triggered; exit 0; }
|
||||
done
|
||||
echo start-failed
|
||||
exit 1
|
||||
PHASE2_TRIGGER
|
||||
)
|
||||
# Failure patterns are matched first, and they echo a path back. An owner whose
|
||||
# SCRIPTS_DIR happened to contain the word "triggered" would otherwise satisfy a
|
||||
# leading *triggered* glob and report success for a launch that never happened.
|
||||
case "$_phase2_out" in
|
||||
missing-runner:*)
|
||||
error "Phase 2 not started — no run_job.sh at ${_phase2_out#missing-runner:} on $OWNER"
|
||||
;;
|
||||
missing-script:*)
|
||||
error "Phase 2 not started — no partnership_onboard.sh at ${_phase2_out#missing-script:} on $OWNER"
|
||||
;;
|
||||
start-failed)
|
||||
error "Phase 2 launch on $OWNER did not produce a job record — check run_job.sh there"
|
||||
;;
|
||||
triggered)
|
||||
echo "Phase 2 triggered on $OWNER ✅"
|
||||
log "Watch on $OWNER: Scheduler tab, or tail -f /var/log/varaverk/Partnership/partnership_onboard.log"
|
||||
PHASE2_TRIGGERED=true
|
||||
;;
|
||||
*)
|
||||
warn "Could not auto-trigger Phase 2 on $OWNER"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
else
|
||||
warn "Cannot resolve $OWNER Tailscale IP"
|
||||
@@ -356,10 +486,129 @@ MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||||
log "Mirror: $MIRROR ($MIRROR_IP)"
|
||||
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
||||
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
||||
[[ "$PHASE3_ONLY" == true ]] && log "Mode: Phase 3 only (media seed)"
|
||||
echo ""
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PHASE 3: MEDIA SEED ───────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# The partnership is finished at the end of Phase 2 — connected, running, and saying so. Phase 3
|
||||
# is not part of it. It is the separate decision to fill the partner's disks by rsync, and it is
|
||||
# operator-triggered because a first seed of a full library is weeks of transfer and tens of
|
||||
# terabytes of somebody else's free space.
|
||||
#
|
||||
# The seed used to be a step inside Phase 2. Inline it held the onboard open for the entire
|
||||
# transfer, which meant the phase-2 flag, the conf push, discovery and container grouping all
|
||||
# waited on it, and both hosts described an unfinished onboard over a partnership that was
|
||||
# already live. Detaching it fixed the blocking but left the seed starting on its own, which is
|
||||
# still the wrong default: the machine decided to move 28 TB because an onboard succeeded.
|
||||
#
|
||||
# With MEDIA_SEED_ENABLED=false there is no Phase 3 at all and the model is two phases. That is
|
||||
# the point of the toggle — not a seed that runs and does nothing, an absent phase.
|
||||
# ==============================================================================================
|
||||
if [[ "$PHASE3_ONLY" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SYNC PHASE 3 — MEDIA SEED — $MY_ID → $MIRROR ━━━━━"
|
||||
echo ""
|
||||
|
||||
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||||
_seed_gate=$(grep -m1 -E '^[[:space:]]*MEDIA_SEED_ENABLED=' "$_master_conf" 2>/dev/null \
|
||||
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
||||
|
||||
if [[ -n "$_seed_gate" && "$_seed_gate" != "true" ]]; then
|
||||
error "MEDIA_SEED_ENABLED is '$_seed_gate' — there is no Phase 3 on this host"
|
||||
error " Onboarding is two phases while seeding is off. Arm the toggle in master.conf"
|
||||
error " if you want the partner's library filled by rsync."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
_setup_db="$(platform_setup_db_path)"
|
||||
if ! grep -q "^${MIRROR_ID}_PHASE2_DONE=true" "$_setup_db" 2>/dev/null; then
|
||||
error "$MIRROR has not completed Phase 2 — seeding a partner that is not onboarded"
|
||||
error " would push media at a host with no containers to serve it. Finish Phase 2 first."
|
||||
exit 1
|
||||
fi
|
||||
unset _setup_db
|
||||
|
||||
# Seeding posture, set before the transfer rather than after it. Tier 1 has to be open or
|
||||
# rsync.sh refuses every share; every Tier 2 gate is closed so the scheduled orchestrators
|
||||
# are not competing for the same link and the same disks for the next several weeks. This is
|
||||
# also the posture the seed leaves behind — global on, per-orchestrator off — so finishing
|
||||
# Phase 3 does not quietly hand the schedule back.
|
||||
echo "━━━ Gate posture ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set RSYNC_ENABLED true and every Tier 2 rsync gate false"
|
||||
else
|
||||
cp -a "$_master_conf" "${_master_conf}.bak-phase3-$(date +%Y%m%d-%H%M%S)"
|
||||
set_conf_bool RSYNC_ENABLED "true" "$_master_conf" \
|
||||
|| { error "Could not open Tier 1 — the seed would move nothing"; exit 1; }
|
||||
for _gate in CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED \
|
||||
WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED; do
|
||||
set_conf_bool "$_gate" "false" "$_master_conf" \
|
||||
|| warn "Could not close $_gate — a scheduled sync may run beside the seed"
|
||||
done
|
||||
unset _gate
|
||||
echo " Tier 1 open, every Tier 2 gate closed ✅"
|
||||
fi
|
||||
unset _master_conf _seed_gate
|
||||
|
||||
echo ""
|
||||
echo "━━━ Dispatch ━━━"
|
||||
_seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh"
|
||||
_runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would dispatch Rsync/media_seed.sh"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ! -f "$_seed_script" || ! -f "$_runner" ]]; then
|
||||
error "Seed runner not found at $_seed_script"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# setsid, not nohup + &: the seed must outlive this script and must not share its process
|
||||
# group, or stopping Phase 3 takes the transfer down with it.
|
||||
_seed_flags=()
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
||||
setsid bash "$_runner" "Rsync/media_seed.sh" "$_seed_script" "${_seed_flags[@]}" \
|
||||
>/dev/null 2>&1 < /dev/null &
|
||||
disown 2>/dev/null || true
|
||||
|
||||
# Report the record, not the launch. run_job.sh writes its stat file before running the
|
||||
# script, so a fresh "running" record is the difference between dispatched and attempted.
|
||||
_seed_stat="/var/log/varaverk/Rsync/media_seed.json"
|
||||
_dispatched=false
|
||||
for _i in $(seq 1 10); do
|
||||
if [[ -f "$_seed_stat" ]] \
|
||||
&& grep -q '"status":"running"' "$_seed_stat" 2>/dev/null \
|
||||
&& (( $(date +%s) - $(stat -c %Y "$_seed_stat") < 60 )); then
|
||||
_dispatched=true
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 3 SUMMARY ━━━━━"
|
||||
if [[ "$_dispatched" == true ]]; then
|
||||
echo " Seed: running as job Rsync/media_seed.sh ✅"
|
||||
echo " Shares: ${#DAILY_SYNC_SHARES[@]}"
|
||||
echo " Gates: Tier 1 open · every Tier 2 closed"
|
||||
echo ""
|
||||
echo " A first full seed runs for days. Watch it on the Partnership tab; the"
|
||||
echo " partnership itself has been live since Phase 2 and does not depend on this."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
error " Seed did not start — check /var/log/varaverk/Rsync/media_seed.log"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STEP_SSH_OK=false
|
||||
STEP_NETWORK_OK=false
|
||||
LOCAL_SETUP_OK=true # partnership_manager --local-only; the summary claimed done ✅ regardless
|
||||
PHASE1_NET_OK=false # Phase 1 only — network created on the mirror before any deploy
|
||||
PHASE1_CACHE_OK=false # Phase 1 only — our conf pushed into the mirror's RAM cache
|
||||
STEP_STOP_AUTH_OK=true
|
||||
STEP_AUTH_OK=true
|
||||
AUTH_DEPLOYED=0
|
||||
@@ -375,8 +624,6 @@ SERVICES_FAILED=0
|
||||
ONBOARD_OK=false
|
||||
ARR_SYNC_OK=false
|
||||
WEBHOOK_SETUP_OK=false
|
||||
MEDIA_SEED_OK=false
|
||||
MEDIA_SEED_COUNT=0
|
||||
WEBHOOK_LISTENER_OK=false
|
||||
MASTER_PUSH_OK=false
|
||||
|
||||
@@ -443,15 +690,17 @@ if [[ "$PHASE1_ONLY" == true ]]; then
|
||||
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
||||
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
||||
LOCAL_SETUP_OK=false
|
||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
||||
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
||||
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
||||
echo " HOST1 setup: done ✅"
|
||||
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " ACTION NEEDED: install the key on $MIRROR:"
|
||||
@@ -482,11 +731,72 @@ if [[ "$PHASE1_ONLY" == true ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Phase 1 — Docker network on the mirror ────────────────────────────────────────────────
|
||||
# Here, not only in Step 1b, because a --phase1-only run exits above and never reaches it.
|
||||
# The mirror needs docker, not Varaverk, so this works before the plugin is installed — and
|
||||
# creating the network now means it is in place long before the first container is deployed
|
||||
# against it. Deploying against a missing network is what left twelve containers stuck in
|
||||
# `Created`, so the earliest safe moment is the right one.
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — Docker Network ($MIRROR) ━━━"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create any networks the stack templates reference on $MIRROR"
|
||||
PHASE1_NET_OK=true
|
||||
elif ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||
PHASE1_NET_OK=true
|
||||
else
|
||||
warn "Network prep incomplete on $MIRROR — Step 1b retries this during Phase 2"
|
||||
fi
|
||||
|
||||
# ── Phase 1 — Partner conf cache ──────────────────────────────────────────────────────────
|
||||
# CONF_SYNC_ENABLED is armed here rather than waiting for Step 1d. It is the safe one of the
|
||||
# three gates — it moves no data, it copies each side's host*.conf into the other's tmpfs so
|
||||
# partner vars resolve — and the moment SSH works is the moment that should start. Leaving it
|
||||
# until Phase 2 meant conf_sync.sh, which sources the conf fresh in its own process, exited
|
||||
# on a closed gate every time it was called before then.
|
||||
#
|
||||
# Push always; pull only if the mirror actually has a conf yet. Before HOST2 installs Varaverk
|
||||
# there is nothing to pull, and an unconditional pull would count a failure and notify about
|
||||
# a condition that is simply "the mirror is not installed". The pull lands on the re-run after the
|
||||
# install — the same --phase1-only --skip-ssh the operator uses to push conf again.
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — Partner Conf Cache ━━━"
|
||||
_conf_sync_script="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would arm CONF_SYNC_ENABLED and cache confs with $MIRROR"
|
||||
PHASE1_CACHE_OK=true
|
||||
elif [[ ! -f "$_conf_sync_script" ]]; then
|
||||
warn "conf_sync.sh not found — skipping partner conf cache"
|
||||
else
|
||||
set_conf_bool CONF_SYNC_ENABLED "true" "$SCRIPTS_ROOT/Configurations/master.conf" \
|
||||
|| warn "Could not arm CONF_SYNC_ENABLED — cache step may no-op"
|
||||
|
||||
if bash "$_conf_sync_script" --push-only; then
|
||||
PHASE1_CACHE_OK=true
|
||||
else
|
||||
warn "Could not push our conf to $MIRROR"
|
||||
fi
|
||||
|
||||
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$MIRROR_IP" \
|
||||
"[ -f '${_mirror_sd}/Configurations/${MIRROR_ID,,}.conf' ]" 2>/dev/null; then
|
||||
bash "$_conf_sync_script" --pull-only || warn "Could not pull ${MIRROR_ID,,}.conf from $MIRROR"
|
||||
else
|
||||
echo " $MIRROR has no ${MIRROR_ID,,}.conf yet — it will cache once Varaverk is installed there"
|
||||
fi
|
||||
unset _mirror_sd
|
||||
fi
|
||||
unset _conf_sync_script
|
||||
|
||||
# HOST1 local setup — runs immediately without needing HOST2
|
||||
echo ""
|
||||
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
||||
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
||||
LOCAL_SETUP_OK=false
|
||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
||||
|
||||
@@ -495,11 +805,29 @@ if [[ "$PHASE1_ONLY" == true ]]; then
|
||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
||||
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
||||
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
||||
echo " HOST1 setup: done ✅"
|
||||
echo " Network: $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )"
|
||||
echo " Conf cache: $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )"
|
||||
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
||||
echo " The wizard will detect the pushed conf and take the correct path."
|
||||
# The push no longer waits for a Varaverk install on the far side. It resolves the partner's
|
||||
# conf directory across both layouts and creates the internal one if neither exists, so the
|
||||
# conf arrives BEFORE the plugin — which is the order that makes it useful. The .plg only
|
||||
# seeds master.conf from the template when none is present, so what lands here survives the
|
||||
# install and the wizard reads its identity straight out of it.
|
||||
if [[ "$CONF_PUSH_OK" == true ]]; then
|
||||
echo " master.conf is on $MIRROR — the wizard will find it and take the partner path,"
|
||||
echo " already knowing $MY_ID and $MIRROR_ID. If the operator picks flash storage there,"
|
||||
echo " storage_migrate.sh moves the conf to appdata with the rest of the install."
|
||||
else
|
||||
echo " master.conf was NOT delivered. Phase 1 seeds it into a bare host, so this is a"
|
||||
echo " real failure, not the pre-install state — check SSH and that /boot is writable"
|
||||
echo " on $MIRROR. Retry with:"
|
||||
echo " • bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
||||
echo " • or push from Scheduler → master.conf → Save Conf"
|
||||
echo " • or, once the plugin is installed, 'Pull from HOST1' on HOST2's Setup tab"
|
||||
fi
|
||||
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
@@ -510,18 +838,45 @@ fi
|
||||
echo ""
|
||||
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
|
||||
|
||||
_net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh"
|
||||
# Two halves, and the first is the one that matters for a fresh mirror.
|
||||
#
|
||||
# ensure_stack_networks_on_remote reads the networks out of the XMLs this onboard is about to
|
||||
# push and creates any that are missing on the mirror. It does not consult the mirror's conf,
|
||||
# because on a fresh node that array is the template default — a single commented-out entry —
|
||||
# and an empty list is indistinguishable from "no networks needed". The result was every
|
||||
# container in both stacks created against a network that did not exist.
|
||||
#
|
||||
# docker_network_connect.sh still runs afterwards: it is what *connects* the mirror's own
|
||||
# listed containers to its own listed networks, which is a different job and remains the
|
||||
# mirror's to declare.
|
||||
if ! ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||
warn "One or more stack networks could not be prepared on $MIRROR — deploys below may fail"
|
||||
fi
|
||||
|
||||
# Resolve the path on the mirror, not here. This used SCRIPTS_ROOT — the OWNER's install path —
|
||||
# to name a script it then runs over SSH on the MIRROR. That only holds while both hosts install
|
||||
# to the same place, and an appdata-mode mirror is at /mnt/user/appdata/Varaverk, so bash was
|
||||
# handed a path that does not exist there and Step 1b failed on every appdata install. The
|
||||
# warning even printed the owner's path and told the operator to go check it on the mirror.
|
||||
#
|
||||
# Same fix as Steps 11 and 12, which already resolve the remote layout this way.
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
|
||||
STEP_NETWORK_OK=true
|
||||
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||
"bash '$_net_script'" 2>/dev/null; then
|
||||
echo "Docker network ready on $MIRROR ✅"
|
||||
STEP_NETWORK_OK=true
|
||||
else
|
||||
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
||||
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
||||
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||
_net_script="${_mirror_sd}/Docker_Essentials/docker_network_connect.sh"
|
||||
if timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$MIRROR_IP" \
|
||||
"[ -f '$_net_script' ] || { echo missing; exit 127; }; bash '$_net_script'" 2>/dev/null; then
|
||||
echo "Docker network ready on $MIRROR ✅"
|
||||
STEP_NETWORK_OK=true
|
||||
else
|
||||
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
||||
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
||||
fi
|
||||
unset _mirror_sd
|
||||
fi
|
||||
|
||||
# ── Step 1c: Share setup ─────────────────────────────────────────────────────────────────────
|
||||
@@ -536,6 +891,135 @@ else
|
||||
bash "$SCRIPT_DIR/share_setup.sh"
|
||||
fi
|
||||
|
||||
# ── Step 1c2: Permit the mirror ───────────────────────────────────────────────────────────────
|
||||
# A previous offboard blocklists the mirror, and rsync.sh refuses any host on that list outright
|
||||
# — deliberately, so a stale cron cannot keep syncing to a partner that has been removed.
|
||||
#
|
||||
# The blocklist was cleared inside Step 8, which was fine while every rsync in the onboard came
|
||||
# after it. Step 1e does not: it runs before the containers are deployed, which is the whole
|
||||
# point of it. On a re-onboard that ordering means the provisioning sync is refused, the auth
|
||||
# stack comes up against empty directories, and the step built to prevent exactly that reports
|
||||
# a failure nobody can explain from its own output.
|
||||
#
|
||||
# Un-blocking here is not a loosening. The offboard's own summary says "re-onboard to permit
|
||||
# access again" — an onboard is the event that permits it, and this is where the onboard starts
|
||||
# needing it. Step 8 still runs its own clear; removing an absent entry is a no-op.
|
||||
echo ""
|
||||
echo "━━━ Step 1c2 — Permit $MIRROR ━━━"
|
||||
|
||||
UNBLOCK_OK=true
|
||||
_blocklist="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}"
|
||||
if [[ ! -f "$_blocklist" ]] || ! grep -q "^${MIRROR}|" "$_blocklist" 2>/dev/null; then
|
||||
echo " $MIRROR is not blocklisted ✅"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove $MIRROR from the partnership blocklist"
|
||||
else
|
||||
sed -i "/^${MIRROR}|/d" "$_blocklist" 2>/dev/null
|
||||
if grep -q "^${MIRROR}|" "$_blocklist" 2>/dev/null; then
|
||||
error "Could not remove $MIRROR from $_blocklist — rsync.sh will refuse it"
|
||||
UNBLOCK_OK=false
|
||||
else
|
||||
echo " Removed $MIRROR from the blocklist ✅"
|
||||
fi
|
||||
fi
|
||||
unset _blocklist
|
||||
|
||||
# ── Step 1d: Sync gates ───────────────────────────────────────────────────────────────────────
|
||||
# This was Step 9c, after the containers were already deployed. It has to run here instead,
|
||||
# because Step 1e below is itself an rsync and Tier 1 stops every rsync — arming afterwards
|
||||
# meant the provisioning sync exited cleanly having moved nothing.
|
||||
#
|
||||
# master.conf.template ships a fresh node inert. A successful Phase 1 is what makes Tier 1,
|
||||
# CONF_SYNC and ARR_SYNC true; without this the defaults were a one-way door and somebody had
|
||||
# to remember to hand-edit three toggles on the machine where forgetting looks exactly like
|
||||
# everything working.
|
||||
#
|
||||
# Tier 2 is now set false rather than "left as configured". Tier 1 opening is what makes the
|
||||
# scheduled jobs live, and a node that was onboarded ten seconds ago is not a node anyone has
|
||||
# checked yet — free space, share layout, what the partner actually holds. Arriving connected
|
||||
# and idle is the useful state; the operator arms each orchestrator when they mean to.
|
||||
# MEDIA_SEED_ENABLED is not touched here at all — see Step 13.
|
||||
#
|
||||
# Owner only — the mirror receives these values in the Step 10 push rather than deciding.
|
||||
ARM_OK=true
|
||||
_VV_ARM_ON=(RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED)
|
||||
_VV_ARM_OFF=(CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED
|
||||
WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED)
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 1d — Sync Gates ━━━"
|
||||
|
||||
if [[ "$SKIP_ARM" == true ]]; then
|
||||
echo " --no-arm — leaving the sync gates as they are"
|
||||
warn " Step 1e needs Tier 1 open; with it closed the auth appdata will not be provisioned"
|
||||
elif [[ "$AM_OWNER" != true ]]; then
|
||||
echo " mirror — the owner's push decides these"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would set ${_VV_ARM_ON[*]} true and ${_VV_ARM_OFF[*]} false"
|
||||
else
|
||||
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||||
if [[ ! -f "$_master_conf" ]]; then
|
||||
warn "master.conf not found at $_master_conf — gates left closed"
|
||||
ARM_OK=false
|
||||
else
|
||||
cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)"
|
||||
for _gate in "${_VV_ARM_ON[@]}"; do set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false; done
|
||||
for _gate in "${_VV_ARM_OFF[@]}"; do set_conf_bool "$_gate" "false" "$_master_conf" || ARM_OK=false; done
|
||||
unset _gate
|
||||
echo " Tier 1 open: ${_VV_ARM_ON[*]}"
|
||||
echo " Tier 2 off: every scheduled rsync — arm them yourself when you are ready"
|
||||
fi
|
||||
unset _master_conf
|
||||
fi
|
||||
|
||||
# ── Step 1e: Provision the auth stack's appdata ───────────────────────────────────────────────
|
||||
# The one rsync an onboard actually needs, and it has to land before Step 3 creates the
|
||||
# containers that read it. Deploying first meant Authelia, Lldap, NPM and both databases came up
|
||||
# on the mirror against empty directories and initialised themselves from nothing — so the
|
||||
# mirror's auth stack was a fresh install wearing the owner's container names.
|
||||
#
|
||||
# The critical-data profile, not critical-fallback. critical-data stops the auth containers on
|
||||
# both sides for the duration, which costs the owner its auth for about a minute at 588 MB, and
|
||||
# buys a consistent copy. critical-fallback is the dirty variant that keeps auth running, and a
|
||||
# dirty copy is exactly what broke Redis and MariaDB on the mirror before: the manifest and the
|
||||
# binlog index were copied without the files they name, and MariaDB hides that — the container
|
||||
# reads Up while mysqld_safe restarts the dead database every few seconds.
|
||||
#
|
||||
# Media is not seeded here or anywhere in Phase 2. This share is 588 MB and the containers do
|
||||
# not start correctly without it; a media library is terabytes and nothing waits on it.
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Step 1e — Auth Appdata Provision ━━━"
|
||||
|
||||
APPDATA_PROVISION_OK=false
|
||||
if [[ "$SKIP_APPDATA_PROVISION" == true ]]; then
|
||||
warn "Skipping (--skip-appdata-provision) — the mirror's auth stack will start empty"
|
||||
elif [[ "${#PARTNERSHIP_PROVISION_SHARES[@]}" -eq 0 ]]; then
|
||||
warn "PARTNERSHIP_PROVISION_SHARES is empty — nothing to provision"
|
||||
warn " Set it in host${MY_ID: -1}.conf, or the mirror's auth stack starts from nothing"
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would sync ${PARTNERSHIP_PROVISION_SHARES[*]} to $MIRROR"
|
||||
APPDATA_PROVISION_OK=true
|
||||
else
|
||||
_prov_rc=0
|
||||
for _share in "${PARTNERSHIP_PROVISION_SHARES[@]}"; do
|
||||
echo " Provisioning: $_share"
|
||||
_prov_flags=()
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _prov_flags+=(--log)
|
||||
if ! bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_prov_flags[@]}"; then
|
||||
warn " Failed: $_share"
|
||||
_prov_rc=1
|
||||
fi
|
||||
done
|
||||
unset _share _prov_flags
|
||||
if [[ "$_prov_rc" -eq 0 ]]; then
|
||||
echo "Auth appdata provisioned to $MIRROR ✅"
|
||||
APPDATA_PROVISION_OK=true
|
||||
else
|
||||
warn "Auth appdata NOT fully provisioned — the containers below will start against"
|
||||
warn " whatever is already on $MIRROR, which on a fresh node is nothing"
|
||||
fi
|
||||
unset _prov_rc
|
||||
fi
|
||||
|
||||
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
||||
@@ -672,46 +1156,11 @@ else
|
||||
fi
|
||||
unset _webhook_script
|
||||
|
||||
# ── Step 9c: Seed media shares on mirror ─────────────────────────────────────────────────────
|
||||
# arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but
|
||||
# have no files on disk. Without this rsync, every imported item looks missing and arrs
|
||||
# will immediately queue searches. --seed skips the empty-remote guard and does a clean push.
|
||||
echo ""
|
||||
echo "━━━ Step 9c — Media Share Seed ━━━"
|
||||
|
||||
if [[ "$SKIP_MEDIA_SEED" == true ]]; then
|
||||
warn "Skipping (--skip-media-seed)"
|
||||
elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
|
||||
warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed"
|
||||
warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually"
|
||||
else
|
||||
echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while"
|
||||
_rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh"
|
||||
_seed_flags=(--seed)
|
||||
[[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run)
|
||||
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
||||
for _share in "${DAILY_SYNC_SHARES[@]}"; do
|
||||
echo " Seeding: $_share"
|
||||
if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then
|
||||
(( MEDIA_SEED_COUNT++ )) || true
|
||||
else
|
||||
warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
|
||||
fi
|
||||
done
|
||||
if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then
|
||||
echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅"
|
||||
MEDIA_SEED_OK=true
|
||||
else
|
||||
warn "Media seed: no shares completed — check errors above"
|
||||
fi
|
||||
unset _rsync_script _seed_flags _share
|
||||
fi
|
||||
|
||||
# ── Step 9d: Start webhook listener on mirror ─────────────────────────────────────────────────
|
||||
# ── Step 9e: Start webhook listener on mirror ─────────────────────────────────────────────────
|
||||
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
|
||||
# already running — kick it now so events are captured immediately after onboard.
|
||||
echo ""
|
||||
echo "━━━ Step 9d — Webhook Listener (Mirror) ━━━"
|
||||
echo "━━━ Step 9e — Webhook Listener (Mirror) ━━━"
|
||||
|
||||
_listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh"
|
||||
if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then
|
||||
@@ -756,6 +1205,133 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Step 11: Service discovery on the mirror ──────────────────────────────────────────────────
|
||||
# Deliberately last. conf_populate.sh fills host*.conf from what it can actually find running —
|
||||
# arr API keys, container names, URLs — and until Step 3 and Step 5 deployed the auth and arr
|
||||
# stacks there was nothing on the mirror to find. The wizard runs it during first-run setup,
|
||||
# which on a fresh mirror is precisely the moment the machine is still empty, so everything it
|
||||
# could have discovered was discovered as absent.
|
||||
#
|
||||
# No --overwrite: it only fills blanks, so anything the operator set by hand survives. --no-push
|
||||
# because Step 10 above has just pushed conf; letting discovery push again would race it.
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 11 — Service Discovery ($MIRROR) ━━━"
|
||||
|
||||
POPULATE_OK=false
|
||||
# MIRROR_IP, not MIRROR_REACHABLE — the latter is partnership_offboard.sh's variable and does not
|
||||
# exist in this script, so the test was always true against an empty string and Step 11 reported
|
||||
# "skipped (unreachable)" on a mirror it had just deployed twelve containers to.
|
||||
if [[ -z "${MIRROR_IP:-}" ]]; then
|
||||
warn "$MIRROR has no resolved IP — skipping discovery, run Deployment/conf_populate.sh there later"
|
||||
POPULATE_OK=skipped
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run conf_populate.sh --no-push on $MIRROR"
|
||||
POPULATE_OK=true
|
||||
else
|
||||
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||
_pop_script="${_mirror_sd}/Deployment/conf_populate.sh"
|
||||
if timeout 180 ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$MIRROR_IP" \
|
||||
"[ -f '$_pop_script' ] || { echo missing; exit 127; }; bash '$_pop_script' --no-push" 2>/dev/null; then
|
||||
echo "Discovery complete on $MIRROR ✅"
|
||||
POPULATE_OK=true
|
||||
else
|
||||
warn "Discovery failed on $MIRROR — run $_pop_script there by hand"
|
||||
fi
|
||||
unset _mirror_sd _pop_script
|
||||
fi
|
||||
|
||||
# ── Step 12: Group our containers on the mirror ───────────────────────────────────────────────
|
||||
# The mirror now runs a dozen containers that are ours, scattered among its own. This files them
|
||||
# under one folder named after us — "<OwnerShort>-Fallback" — matching the convention the owner
|
||||
# already keeps for the mirror's containers.
|
||||
#
|
||||
# The icon is resolved HERE and passed over, not looked up there. It comes from the closest Emby
|
||||
# user to our own name, and the mirror has neither our Emby key nor necessarily an Emby at all —
|
||||
# so a lookup on that side would find nothing and the folder would come up blank.
|
||||
#
|
||||
# Not fatal in any direction: folder.view3 absent on the mirror is a clean skip, and a folder
|
||||
# without a picture is still a folder.
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 12 — Container Grouping ($MIRROR) ━━━"
|
||||
|
||||
FOLDER_OK=false
|
||||
if [[ -z "${MIRROR_IP:-}" ]]; then
|
||||
warn "$MIRROR has no resolved IP — skipping container grouping"
|
||||
FOLDER_OK=skipped
|
||||
else
|
||||
mapfile -t _deployed < <(deployed_stack_container_names)
|
||||
_deployed_csv=$(IFS=,; echo "${_deployed[*]}")
|
||||
if [[ -z "$_deployed_csv" ]]; then
|
||||
log "No stack templates resolved to container names — nothing to group"
|
||||
FOLDER_OK=skipped
|
||||
elif [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would create ${MY_ID}-named fallback folder on $MIRROR with: $_deployed_csv"
|
||||
FOLDER_OK=true
|
||||
else
|
||||
_ff_local="$SCRIPTS_ROOT/Plugin/$PLATFORM/Tools/fallback_folder.php"
|
||||
_mf_local="$SCRIPTS_ROOT/Plugin/$PLATFORM/Tools/mirror_folders.php"
|
||||
_icon=$(php "$_ff_local" --host="$MY_ID" --icon-only 2>/dev/null || true)
|
||||
[[ -z "$_icon" ]] && log "No icon resolved for $MY_ID — folder will be created without one"
|
||||
|
||||
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||
_ff_remote="${_mirror_sd}/Plugin/${PLATFORM}/Tools/fallback_folder.php"
|
||||
_mf_remote="${_mirror_sd}/Plugin/${PLATFORM}/Tools/mirror_folders.php"
|
||||
|
||||
# ── 12a: reproduce our own folder layout on the mirror ────────────────────────────────
|
||||
# The deployed stacks run on the mirror continuously — they are not failover coverage —
|
||||
# so they belong on the same shelves they occupy here: Sonarr in "Arrs Stack", NPM and
|
||||
# Lldap in "Networking", the databases in "Databases". Filing all of them under
|
||||
# "<Owner>-Fallback", which is what this step used to do, records whose they are and
|
||||
# nothing about what they do.
|
||||
#
|
||||
# The plan is computed here, where the owner's folder.view3 layout lives, and applied
|
||||
# there. Whatever the layout does not account for comes back as "unfiled" and is what
|
||||
# the fallback folder is actually for.
|
||||
_unfiled="$_deployed_csv"
|
||||
if [[ -f "$_mf_local" ]]; then
|
||||
# Comma-joined: the flag takes a CSV, and "${arr[*]}" would join on spaces — which
|
||||
# also happen to appear inside folder names, so the plan must never be reflowed.
|
||||
_fbonly=$(IFS=,; printf '%s' "${PARTNERSHIP_FALLBACK_ONLY[*]:-}")
|
||||
_plan=$(php "$_mf_local" --export --containers="$_deployed_csv" \
|
||||
--fallback-only="$_fbonly" 2>/dev/null)
|
||||
if [[ -n "$_plan" ]]; then
|
||||
if printf '%s' "$_plan" | timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$MIRROR_IP" \
|
||||
"[ -f '$_mf_remote' ] || exit 127; php '$_mf_remote' --import" 2>/dev/null; then
|
||||
_unfiled=$(printf '%s' "$_plan" | php -r \
|
||||
'echo implode(",", json_decode(stream_get_contents(STDIN),true)["unfiled"] ?? []);' 2>/dev/null)
|
||||
else
|
||||
warn "Could not mirror the folder layout to $MIRROR — falling back to one folder"
|
||||
fi
|
||||
fi
|
||||
unset _plan _fbonly
|
||||
fi
|
||||
|
||||
# ── 12b: the fallback folder gets only what is genuinely fallback ─────────────────────
|
||||
# Unfiled containers plus anything named in PARTNERSHIP_FALLBACK_ONLY. Empty is the
|
||||
# normal, correct outcome when every deployed container has a home in the layout.
|
||||
if [[ -z "$_unfiled" ]]; then
|
||||
echo " Folder layout mirrored — nothing left for ${MY_ID}-Fallback ✅"
|
||||
FOLDER_OK=true
|
||||
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$MIRROR_IP" \
|
||||
"[ -f '$_ff_remote' ] || { echo missing; exit 127; }
|
||||
php '$_ff_remote' --host=$(printf '%q' "$MY_ID") \
|
||||
--containers=$(printf '%q' "$_unfiled") \
|
||||
--icon=$(printf '%q' "$_icon")" 2>/dev/null; then
|
||||
FOLDER_OK=true
|
||||
else
|
||||
warn "Could not group containers on $MIRROR — run $_ff_remote there by hand"
|
||||
fi
|
||||
unset _ff_local _ff_remote _mf_local _mf_remote _mirror_sd _icon _unfiled
|
||||
fi
|
||||
unset _deployed _deployed_csv
|
||||
fi
|
||||
|
||||
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
||||
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
||||
|
||||
@@ -773,6 +1349,9 @@ _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||||
|
||||
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
||||
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
|
||||
echo " Step 1c2— Permit mirror: $(_ok "$UNBLOCK_OK")"
|
||||
echo " Step 1d — Sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$AM_OWNER" != true ]] && echo "mirror — set by owner" || _ok "$ARM_OK"; } )"
|
||||
echo " Step 1e — Auth appdata: $( [[ "$SKIP_APPDATA_PROVISION" == true ]] && echo "skipped" || _ok "$APPDATA_PROVISION_OK" )"
|
||||
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
||||
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
||||
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
||||
@@ -782,9 +1361,10 @@ echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] &&
|
||||
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
||||
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
||||
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
|
||||
echo " Step 9c — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" || echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")" )"
|
||||
echo " Step 9d — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
||||
echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
||||
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
||||
echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )"
|
||||
echo " Step 12 — Grouping: $( [[ "$FOLDER_OK" == skipped ]] && echo "skipped" || _ok "$FOLDER_OK" )"
|
||||
echo ""
|
||||
|
||||
if [[ "$ONBOARD_OK" == true ]]; then
|
||||
|
||||
@@ -12,7 +12,13 @@
|
||||
#
|
||||
# Key named after this server: hostname lowercased, unraid- prefix stripped.
|
||||
# unRAID-Gmer4Lfe → gmer4lfe_rsync_automation
|
||||
# unRAID-Jayred365 → jayred365_rsync_automation
|
||||
# unRAID-Jayred36 → jayred36_rsync_automation
|
||||
#
|
||||
# The name comes from the OS hostname, never from Tailscale. HOST2 answers to both
|
||||
# `unRAID-Jayred36` (hostname -s — Unraid truncates Server Name to the 15-char NetBIOS limit)
|
||||
# and `unraid-jayred365` (its Tailscale peer name), and only the first one decides this filename.
|
||||
# api/setup.php derives the same path independently when it pulls master.conf, from the same
|
||||
# source — so a key created under the Tailscale spelling is a key neither of them will find.
|
||||
#
|
||||
# Idempotent — skips generation if the key already exists (--force to regenerate).
|
||||
# Updates host*.conf with the key path on success.
|
||||
@@ -449,6 +455,22 @@ else
|
||||
if ssh-copy-id -i "$SSH_PUB_PATH" -o ConnectTimeout="${SSH_TIMEOUT:-15}" \
|
||||
root@"$REMOTE_SERVER" 2>/dev/null; then
|
||||
echo "Public key installed on $REMOTE_SERVER_NAME ✅"
|
||||
|
||||
# Tighten the file ssh-copy-id just wrote, or the key it installed is inert.
|
||||
#
|
||||
# On Unraid /root/.ssh is a symlink to /boot/config/ssh/root, and a file created there
|
||||
# comes out 0666. sshd runs StrictModes by default and refuses to read a group- or
|
||||
# world-writable authorized_keys — it logs "Authentication refused: bad ownership or
|
||||
# modes" and falls through to password auth, which BatchMode then declines.
|
||||
#
|
||||
# The failure is silent from this end: ssh-copy-id succeeds and reports success, the key
|
||||
# is present and byte-correct, and every later connection is still refused. Observed
|
||||
# exactly that — key installed at 07:15:12, onboard refused at 07:15:43, blobs identical.
|
||||
ssh -o ConnectTimeout="${SSH_TIMEOUT:-15}" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
-i "$SSH_KEY_PATH" root@"$REMOTE_SERVER" \
|
||||
'chmod 700 "$(dirname "$(readlink -f ~/.ssh/authorized_keys)")" 2>/dev/null
|
||||
chmod 600 "$(readlink -f ~/.ssh/authorized_keys)" 2>/dev/null' 2>/dev/null \
|
||||
|| warn "Could not tighten authorized_keys on $REMOTE_SERVER_NAME — if auth is refused, chmod 600 it there"
|
||||
else
|
||||
error "ssh-copy-id failed — check that:"
|
||||
error " 1. Remote server is reachable: tailscale status"
|
||||
|
||||
@@ -174,7 +174,12 @@ transform_xml_for_gpu() {
|
||||
local src_xml="$1" gpu_type="$2"
|
||||
|
||||
# Detect GPU-aware XMLs — new style (--gpus "device=) or old style (--runtime=nvidia)
|
||||
if ! grep -qE '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
||||
# -- before the pattern is load-bearing: it begins with "--", so without it grep parses the
|
||||
# pattern as an OPTION, exits 2, and the ! makes this branch always true — the function then
|
||||
# returned the XML untransformed every single time, for every GPU type, with 2>/dev/null
|
||||
# swallowing "invalid option". Every container onboarded to a mirror kept the owner NVIDIA
|
||||
# device UUID and could not start on Intel or AMD hardware. Found 2026-08-23.
|
||||
if ! grep -qE -- '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
||||
echo "$src_xml"
|
||||
return 0
|
||||
fi
|
||||
@@ -279,7 +284,10 @@ deploy_container_from_xml() {
|
||||
[[ "$_transformed_xml" != "$xml_file" ]] && _gpu_tmp="$_transformed_xml"
|
||||
xml_file="$_transformed_xml"
|
||||
|
||||
local name repo network extra privileged
|
||||
local name repo network extra privileged webui icon
|
||||
# WebUI and Icon become Unraid labels below — see the docker create line for why.
|
||||
webui=$( awk 'match($0,/<WebUI>([^<]*)<\/WebUI>/, a){print a[1];exit}' "$xml_file")
|
||||
icon=$( awk 'match($0,/<Icon>([^<]*)<\/Icon>/, a){print a[1];exit}' "$xml_file")
|
||||
name=$( awk 'match($0,/<Name>([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file")
|
||||
repo=$( awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
|
||||
network=$( awk 'match($0,/<Network>([^<]+)<\/Network>/, a){print a[1];exit}' "$xml_file")
|
||||
@@ -318,7 +326,19 @@ deploy_container_from_xml() {
|
||||
printf "docker stop %q 2>/dev/null || true\n" "$name"
|
||||
printf "docker rm %q 2>/dev/null || true\n" "$name"
|
||||
echo ""
|
||||
# Unraid's Docker Manager decides what it owns by label, not by template presence. The
|
||||
# XML is SCPed to the mirror's templates-user above, but without these three the WebGUI
|
||||
# lists the container as third-party: no Edit button, no WebUI link, no icon — the
|
||||
# operator can see it running and cannot do anything with it.
|
||||
#
|
||||
# The values go in verbatim, placeholders and all: Unraid stores the literal
|
||||
# "http://[IP]:[PORT:8989]/..." form in the label and substitutes at render time, so
|
||||
# resolving them here would produce a link that stops being right the moment the
|
||||
# container's port mapping changes.
|
||||
printf "docker create --name %q --restart=unless-stopped" "$name"
|
||||
printf " --label %q" "net.unraid.docker.managed=dockerman"
|
||||
[[ -n "$webui" ]] && printf " --label %q" "net.unraid.docker.webui=${webui}"
|
||||
[[ -n "$icon" ]] && printf " --label %q" "net.unraid.docker.icon=${icon}"
|
||||
[[ -n "$network" ]] && printf " --network=%q" "$network"
|
||||
[[ "$privileged" == "true" ]] && printf " --privileged"
|
||||
[[ -n "$extra" ]] && printf " %s" "$extra"
|
||||
@@ -416,6 +436,112 @@ deploy_xml_stack() {
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Ensure the networks our pushed templates reference exist on a remote host ─────────────────
|
||||
#
|
||||
# The owner deploys the mirror's containers from the owner's own XMLs, and those XMLs name a
|
||||
# network. `docker create` fails outright if that network is missing, so the network has to
|
||||
# exist on the mirror before any stack is deployed.
|
||||
#
|
||||
# This used to be left entirely to docker_network_connect.sh running on the mirror, which
|
||||
# iterates the *mirror's* NETWORK_CONNECT_NETWORKS. host.conf.template ships that array with its
|
||||
# only entry commented out, so on a fresh node it is empty — nothing was created, and every
|
||||
# container in both stacks was created against a network that did not exist and could never
|
||||
# start. Twelve containers stuck in `Created`, reported as "0 deployed, 8 failed" and
|
||||
# "0 deployed, 5 failed" as though each container had its own problem.
|
||||
#
|
||||
# The owner knows what it is about to push, so it derives the requirement from the templates
|
||||
# rather than trusting the mirror's conf to have been filled in.
|
||||
#
|
||||
# Only bridge networks are created. br0 and friends are ipvlan/macvlan bound to real host
|
||||
# hardware — the parent interface cannot be inferred from here, and guessing one would attach
|
||||
# the mirror's containers to the wrong segment.
|
||||
#
|
||||
# Usage: ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
# ==============================================================================================
|
||||
# ==============================================================================================
|
||||
# ── Container names this onboard deploys, read from the templates it deploys them from ────────
|
||||
#
|
||||
# Echoes one name per line. The <Name> element is the same value deploy_xml_stack() passes to
|
||||
# `docker create --name`, so this is the deployed set by construction rather than by asking the
|
||||
# mirror what it ended up with — which would also pick up whatever the mirror already ran.
|
||||
#
|
||||
# Usage: mapfile -t names < <(deployed_stack_container_names)
|
||||
# ==============================================================================================
|
||||
deployed_stack_container_names() {
|
||||
local -a xml_names=()
|
||||
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||
[[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")
|
||||
|
||||
local xml_name xml_file cname
|
||||
for xml_name in "${xml_names[@]}"; do
|
||||
[[ -z "$xml_name" ]] && continue
|
||||
xml_file="${TEMPLATES_DIR}/${xml_name}"
|
||||
[[ -f "$xml_file" ]] || continue
|
||||
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
||||
[[ -n "$cname" ]] && echo "$cname"
|
||||
done
|
||||
}
|
||||
|
||||
ensure_stack_networks_on_remote() {
|
||||
local remote_ip="$1" ssh_key="$2"
|
||||
local -a xml_names=() nets=()
|
||||
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||
[[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")
|
||||
|
||||
local xml_name xml_file net n seen
|
||||
for xml_name in "${xml_names[@]}"; do
|
||||
[[ -z "$xml_name" ]] && continue
|
||||
xml_file="${TEMPLATES_DIR}/${xml_name}"
|
||||
[[ -f "$xml_file" ]] || continue
|
||||
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$xml_file" 2>/dev/null | head -1)
|
||||
net="${net//[[:space:]]/}"
|
||||
# Built-ins exist on every host; br* is host hardware, handled above.
|
||||
case "$net" in ''|bridge|host|none|br[0-9]*) continue ;; esac
|
||||
seen=false
|
||||
for n in "${nets[@]}"; do [[ "$n" == "$net" ]] && { seen=true; break; }; done
|
||||
[[ "$seen" == false ]] && nets+=("$net")
|
||||
done
|
||||
|
||||
if [[ ${#nets[@]} -eq 0 ]]; then
|
||||
log "No custom networks referenced by the pushed templates"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local rc=0 driver
|
||||
for net in "${nets[@]}"; do
|
||||
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" \
|
||||
--format '{{.Driver}}' 2>/dev/null)
|
||||
if [[ -z "$driver" ]]; then
|
||||
warn "$net is referenced by a pushed template but does not exist here either — skipping"
|
||||
rc=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$driver" != "bridge" ]]; then
|
||||
warn "$net is $driver here, not bridge — create it on $MIRROR by hand, its parent interface is host-specific"
|
||||
rc=1
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would ensure network '$net' (bridge) exists on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||
root@"$remote_ip" \
|
||||
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
||||
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null; then
|
||||
echo " network $net (bridge) ready on $MIRROR ✅"
|
||||
else
|
||||
warn " could not ensure network $net on $MIRROR — its containers will not start"
|
||||
rc=1
|
||||
fi
|
||||
done
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Remove owner-deployed containers from a remote host ──────────────────────────────────────
|
||||
#
|
||||
@@ -426,6 +552,7 @@ deploy_xml_stack() {
|
||||
# ==============================================================================================
|
||||
cleanup_deployed_stack_on_remote() {
|
||||
local remote_ip="$1" ssh_key="$2"
|
||||
local _rc=0
|
||||
local -a xml_names=()
|
||||
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||
@@ -471,13 +598,20 @@ cleanup_deployed_stack_on_remote() {
|
||||
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
|
||||
echo " Appdata removed on $MIRROR: $path ✅"
|
||||
else
|
||||
warn " Failed to remove appdata on $MIRROR: $path"
|
||||
_rc=1
|
||||
fi
|
||||
done <<< "$appdata_paths"
|
||||
done
|
||||
# Only appdata failures are counted. The container branch above cannot tell "removal failed"
|
||||
# from "already gone" — both produce no `removed` echo — and an offboard re-run on a
|
||||
# half-finished teardown is a normal case, so treating that as failure would cry wolf.
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -490,6 +624,12 @@ cleanup_deployed_stack_on_remote() {
|
||||
cleanup_deployed_stack_locally() {
|
||||
local owner_ip="$1" ssh_key="$2"
|
||||
local -a xml_names=()
|
||||
# Callers write `cleanup_deployed_stack_locally … || STEP_STACK_CLEANUP_OK=false`, so the
|
||||
# exit status is what the offboard summary prints. Every removal below warns and carries on
|
||||
# — one container that will not die must not abandon the rest of the stack — which meant the
|
||||
# function ended on a `done` and could only ever return 0. Step 3 reported ✅ even when every
|
||||
# docker rm and every rm -rf had failed. Failures are collected here and reported at the end.
|
||||
local _rc=0
|
||||
|
||||
if [[ -n "$owner_ip" ]]; then
|
||||
local -a auth_arr arr_arr
|
||||
@@ -513,8 +653,13 @@ cleanup_deployed_stack_locally() {
|
||||
fi
|
||||
|
||||
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
||||
log "Could not read deployed stack from owner — skipping auth/arr/services cleanup"
|
||||
return 0
|
||||
# Not a success. OWNER_REACHABLE only means a probe answered — the three SSH reads above
|
||||
# can still time out or come back empty, and then nothing was cleaned. The caller's own
|
||||
# unreachable-owner branch sets STEP_STACK_CLEANUP_OK=false for exactly this situation,
|
||||
# so returning 0 here made the summary claim a cleanup that never ran.
|
||||
warn "Could not read deployed stack from owner — auth/arr/services cleanup did not run"
|
||||
warn "Containers will remain — re-run when the owner answers over SSH"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local _local_short
|
||||
@@ -551,17 +696,28 @@ cleanup_deployed_stack_locally() {
|
||||
"$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
||||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
|
||||
_PM_TRAP_STOPPED+=("$cname")
|
||||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1 && \
|
||||
echo " $cname removed ✅" || warn " $cname rm failed"
|
||||
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1; then
|
||||
echo " $cname removed ✅"
|
||||
else
|
||||
warn " $cname rm failed"
|
||||
_rc=1
|
||||
fi
|
||||
else
|
||||
log " $cname not found locally — skipping"
|
||||
fi
|
||||
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
||||
if rm -rf "$path"; then
|
||||
echo " Appdata removed: $path ✅"
|
||||
else
|
||||
warn " Failed to remove: $path"
|
||||
_rc=1
|
||||
fi
|
||||
done <<< "$appdata_paths"
|
||||
done
|
||||
|
||||
return "$_rc"
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -578,11 +734,17 @@ reconfigure_webui() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Match <Name>Authelia</Name>, not "Authelia". Unraid writes the container name as bare XML
|
||||
# text, so the quoted form matched nothing in any template on any host — which is why every
|
||||
# offboard ended with four "template not found ... WebUI needs manual reconfiguration"
|
||||
# warnings and left the mirror's auth WebUIs pointing at the owner it had just left.
|
||||
#
|
||||
# xargs -r so an empty first grep does not run the second one against the whole directory.
|
||||
local template
|
||||
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
|
||||
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
||||
xargs -r grep -l '<Name>$container</Name>' 2>/dev/null | head -1" 2>/dev/null)
|
||||
|
||||
if [[ -z "$template" ]]; then
|
||||
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
||||
|
||||
@@ -103,13 +103,161 @@ once, and the endpoint, the cache writer, and the page all pick it up together.
|
||||
|-------|-------|------|
|
||||
| `pages/` | 11 | One per WebGUI tab — monitor, docker, arrs, fallback, watchdog, rsync, partnership, scheduler, auth, settings, setup |
|
||||
| `api/` | 50 | JSON endpoints the pages poll, plus action endpoints (run a script, stop a job, toggle a flag) |
|
||||
| `include/` | 16 | Shared builders and helpers — `vv_monitor_*`, `vv_arrs_*`, `vv_docker_*`, config read/write, auth |
|
||||
| `include/` | 27 | Shared builders and helpers — `vv_monitor_*`, `vv_arrs_*`, `vv_docker_*`, config read/write, auth |
|
||||
|
||||
**Caching.** Several endpoints serve from `/tmp/vv_cache` (tmpfs) rather than hitting live
|
||||
**The Monitor board is declared, not laid out.** `include/monitor_board.php` holds one array
|
||||
naming every card on the Monitor grid, its width and its order, and generates the whole layout
|
||||
from it — the column ladder, the span clamps at each width, the row-height cap, and the
|
||||
compensation when a conditional card is absent. Column counts are 8/4/2/1 and spans are 1/2/4/8,
|
||||
which is what lets the board re-cut itself at any width with no holes and no hand-placed card.
|
||||
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye, and the row-height cap
|
||||
is conditional on the same arithmetic — it applies only where a rung fits one screen.
|
||||
`pages/monitor.php` carries the card bodies and nothing about where they go; moving a card is
|
||||
moving a line in that array. The page cross-checks the declaration against the cards that
|
||||
actually rendered and says so in the browser if they disagree.
|
||||
|
||||
**Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live
|
||||
APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the
|
||||
cache. A missing or unparseable cache always falls back to a live call, so the cache can never
|
||||
be the reason a page fails to load.
|
||||
|
||||
Anything expensive belongs in that one collection rather than in each consumer. `vv_ai_stats()`
|
||||
costs about a second — mostly waiting on Ollama and `nvidia-smi` — and the AI tab was paying it
|
||||
every 30 seconds per open tab; it is now written once to the `ai` cache and read by the tab, the
|
||||
Monitor row and the dock alike. Polling faster cannot make a figure newer, it only decides how
|
||||
soon a page notices the writer's update.
|
||||
|
||||
> `Tools/api_cache_writer.php` rebuilds the monitor payload **independently** of
|
||||
> `api/monitor.php`, and is what the page normally reads — the endpoint only assembles one on a
|
||||
> cache miss. A key added to the endpoint and not to the writer leaves its card loading forever
|
||||
> on every ordinary page load and working only on the request that happens to miss.
|
||||
|
||||
**One widget, rendered three times.** `include/ai_chat.php` owns the conversation surface —
|
||||
transcript, composer, profile and history pickers, source viewer, stored-chat list — and the AI
|
||||
tab, the Monitor tab's AI row and the Scheduler's right-hand panel all construct it. Every id is
|
||||
composed from a prefix so instances can coexist, and each tears down the previous holder of its
|
||||
prefix, because Unraid swaps tab content by AJAX without unloading the old page's JavaScript.
|
||||
|
||||
A placement chooses how much room it gets and which extra controls it needs. It does not choose
|
||||
the shape, because a chat that rearranges itself per tab is three components wearing one name.
|
||||
The layout is fixed everywhere:
|
||||
|
||||
- **Banner** — title, then `Medium`/`Large` and `Shortcuts` at the right.
|
||||
- **Transcript**.
|
||||
- **Composer**, then one row grouped by what each control acts on: the profile chip at the left
|
||||
for what is *answering*, `New` `Saved` `Ask` centred for what you do to the *conversation*, `⤢`
|
||||
at the right for what you do to the *window*.
|
||||
|
||||
`⤢` and `Medium`/`Large` are deliberately separate: whether the conversation is expanded is an
|
||||
action taken constantly, how much expanded is worth is a preference set once. The shortcuts list
|
||||
is subtracted from the transcript rather than added above it, so opening it never changes the
|
||||
window's total height — which matters on the Scheduler, where that height is a fixed share of a
|
||||
panel that clips.
|
||||
|
||||
The Scheduler panel was exactly that until it was folded in, and it proved the cost: a thread
|
||||
there died on reload while the other two were saved, and its bar had drifted into a different
|
||||
shape from the same control everywhere else. What was genuinely particular to it became options
|
||||
rather than a second implementation — `scope` as a function for a subject that follows the open
|
||||
view, `beforeSend` for the one reply that is recorded rather than asked, `think` for reasoning on
|
||||
diagnosis only, and `setHeights()` for a placement sized as a share of a panel, which is not a
|
||||
number that exists until layout has run.
|
||||
|
||||
**Timezone.** `include/config.php` adopts Unraid's own `timeZone` from `ident.cfg` for the whole
|
||||
PHP layer. PHP here has no `date.timezone` and therefore ran in UTC while the server ran local,
|
||||
so every date this layer produced was offset from every date the shell layer wrote — and the two
|
||||
are compared constantly. `/etc/php.ini` is the wrong fix twice over: `/etc` is a RAM filesystem
|
||||
so the edit dies at reboot, and it would retimezone every other PHP application on the box.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ WHAT GUARDS THE API LAYER ━━━
|
||||
|
||||
Two layers, both supplied by the platform, neither implemented in this plugin.
|
||||
|
||||
**1. nginx session auth.** `/etc/nginx/nginx.conf` applies `satisfy any; allow unix:; deny all;
|
||||
auth_request /auth-request.php;` to everything it serves, `/plugins/` included. A request
|
||||
without a valid WebGUI session never reaches PHP — it is redirected to the login page.
|
||||
|
||||
**2. Unraid's CSRF prepend.** `/etc/php.ini` sets
|
||||
`auto_prepend_file=/usr/local/emhttp/webGui/include/local_prepend.php`, which runs before the
|
||||
first line of any endpoint and, for **every POST**, requires a valid token as either a
|
||||
`csrf_token` body field or an `X-CSRF-Token` header. On a mismatch it logs to syslog and
|
||||
`exit`s. php-fpm inherits this — there is no pool override in `/etc/php-fpm.d/www.conf`.
|
||||
|
||||
Anything that clears both can do everything this plugin can do: stop the array, power off the
|
||||
host, write `master.conf` and push it to every partner, create an lldap user, write and
|
||||
schedule a root-run script. That is the same trust level as the rest of the WebGUI, and it is
|
||||
the intended model.
|
||||
|
||||
### The two consequences that actually matter
|
||||
|
||||
**GET is not covered.** The prepend inspects POSTs only. So any action that *changes something*
|
||||
must be POST — over GET it would run with no token check at all. Four actions were GET and have
|
||||
been converted: `create_api_key.php`, `arrs.php?action=refresh_remote`, `cert.php?action=run`
|
||||
and `setup.php?action=ssh_generate`. **When adding an endpoint the rule is simply: if it changes
|
||||
state, it is POST.** Read-only GETs (`monitor`, `board`, `checklist`, the browse pickers,
|
||||
`api_test.php`) are bounded by the session alone, which is correct for reads.
|
||||
|
||||
**The webGUI's token injector is jQuery-only.** `$.ajaxPrefilter` in dynamix's
|
||||
`HeadInlineJS.php` appends `csrf_token` to jQuery POSTs. Varaverk's pages use native `fetch()`,
|
||||
which it does not touch — so a `fetch()` POST carries no token and the prepend kills it
|
||||
silently: `csrf_terminate()` exits with an empty body, `r.json()` throws on the empty response,
|
||||
and the page's own `.catch()` swallows it. There is no error anywhere except a syslog line.
|
||||
|
||||
`Varaverk.page` therefore installs a small `window.fetch` shim, inline and above the tab
|
||||
content, that attaches `X-CSRF-Token` to same-origin `/plugins/varaverk/` requests. The header
|
||||
form is deliberate: it works for `FormData`, `URLSearchParams` and raw JSON bodies alike, so no
|
||||
call site has to know about it and a new one cannot forget it. A cross-origin page cannot set
|
||||
a custom header without a preflight it will fail, which is what makes it a real defence rather
|
||||
than a formality.
|
||||
|
||||
**Do not construct POSTs outside `fetch()`** — a raw `XMLHttpRequest` or a generated form —
|
||||
without adding the token yourself; the shim only wraps `fetch`. A plain `<form method="POST">`
|
||||
is already covered by a different platform mechanism: dynamix's `BodyInlineJS.php` appends a
|
||||
hidden `csrf_token` input to every form on the page, which is what makes `VaraverkSettings.page`
|
||||
work without any of this.
|
||||
|
||||
### Never send a POST as multipart/form-data
|
||||
|
||||
**`FormData` is banned in this plugin.** A `multipart/form-data` POST to the plugin API *hangs
|
||||
and never completes* on this host — confirmed 2026-08-02.
|
||||
|
||||
The failure is completely silent, which is what makes it dangerous:
|
||||
|
||||
- The browser sends it correctly, valid token and all (verified in the Network tab).
|
||||
- The Network row shows **no status code at all** — not 403, not 500. It never completes.
|
||||
- Server side there is nothing: no CSRF termination in syslog, no fatal in `/var/log/phplog`,
|
||||
and no output from a log statement that is the literal first line of the endpoint.
|
||||
|
||||
Use `URLSearchParams`. `fetch` sets `application/x-www-form-urlencoded` for it automatically,
|
||||
and it has the same `append()` / `set()` API, so it is a drop-in for any payload of strings:
|
||||
|
||||
```js
|
||||
const fd = new URLSearchParams(); // NOT new FormData()
|
||||
fd.append('action', 'save');
|
||||
fetch(url, { method: 'POST', body: fd });
|
||||
```
|
||||
|
||||
The correlation across the plugin was exact: every page using `URLSearchParams` worked, every
|
||||
page using `FormData` hung. All 21 call sites were converted in one pass. Suspected to date from
|
||||
the Unraid 7.3.1→7.3.2 upgrade; root cause in nginx/php-fpm was never identified, only the
|
||||
workaround. If a POST ever hangs with no status code and leaves no server-side trace whatsoever,
|
||||
**suspect the encoding first** — not CSRF, not auth, not the endpoint.
|
||||
|
||||
### api/webhook.php is dead code
|
||||
|
||||
It receives arr download events — but an arr has no WebGUI session and no CSRF token, so it is
|
||||
blocked by *both* layers above and cannot be called at all. The live path is
|
||||
`Arrs_Stack/webhook_listener.js` on `WEBHOOK_PORT`, running outside nginx entirely and
|
||||
validating `WEBHOOK_SECRET` itself. `api/webhook.php` cannot be made to work by adding a secret
|
||||
check — it would have to be served from outside `/plugins/` first. Removing it is the other
|
||||
reasonable option.
|
||||
|
||||
Two endpoints read wider than the rest and are worth knowing about if the session boundary ever
|
||||
moves: `api/api_test.php` returns an API key prefix and the live GraphQL schema, and the browse
|
||||
actions in `api/import_script.php` and `api/manual_sync.php` list directory names anywhere on
|
||||
either host. All are read-only and none return file contents.
|
||||
|
||||
---
|
||||
|
||||
## ━━━ ARRAY LIFECYCLE HOOKS ━━━
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
# master.conf
|
||||
#
|
||||
# MOVER_STOP_TIMEOUT
|
||||
# Seconds between wall warning and SIGTERM. (default: 30)
|
||||
# Seconds between wall warning and SIGTERM. (shipped default: 300)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================== PCIe AER Quiet ================================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Removes listed PCIe devices from the bus at array start so dead hardware stops
|
||||
# flooding syslog with correctable AER errors. Run once via ARRAY_START_SCRIPTS.
|
||||
# Idempotent — silent when the devices are already absent.
|
||||
#
|
||||
# Some onboard controllers throw endless correctable PCIe errors. Correctable
|
||||
# means the link recovered and nothing was lost, but the kernel logs every one,
|
||||
# and on HOST1 that was ~180 lines/day drowning the syslog. Masking the error in
|
||||
# the device's AER registers hides it; `pci=noaer` silences the whole machine
|
||||
# and costs uncorrectable reporting on every device. Removing the device ends it
|
||||
# at the source and leaves AER fully intact everywhere else.
|
||||
#
|
||||
# This only makes sense for hardware nothing is using. The guards below refuse
|
||||
# anything else — see OPERATIONAL SAFEGUARDS.
|
||||
#
|
||||
# A reboot re-enumerates the devices, which is why this reapplies every array
|
||||
# start rather than being a one-time step. It is also the undo: clear the conf
|
||||
# list and reboot.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# One pass over PCIE_QUIET_DEVICES at array start. Each address is looked up under
|
||||
# /sys/bus/pci/devices, checked against the guards below, and removed through the kernel's own
|
||||
# per-device remove attribute. Nothing is masked and no kernel parameter is set.
|
||||
#
|
||||
# Idempotent, and silent when there is nothing to do. An address that is already gone is not an
|
||||
# error — that is the normal state on every array start after the first within one boot.
|
||||
#
|
||||
# Reapplied every array start rather than once, because a reboot re-enumerates the bus and the
|
||||
# devices come back. That is also the undo: clear PCIE_QUIET_DEVICES and reboot.
|
||||
#
|
||||
# PCIE_QUIET_ENABLED gates the whole run before any device is touched.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Refuse Anything In Use
|
||||
# A PCI address is one typo away from the HBA holding the array. The script
|
||||
# removes a device only when the kernel shows it bound to vfio-pci or to no
|
||||
# driver at all. A device with a real driver is skipped loudly, never removed.
|
||||
#
|
||||
# Address Shape Validated Before Any Write
|
||||
# The address goes into a sysfs path. It is matched against the full
|
||||
# DDDD:BB:DD.F form before being used to build one.
|
||||
#
|
||||
# Idempotent Presence Check
|
||||
# A device already gone is not an error — it is the desired state. Silent skip.
|
||||
#
|
||||
# Apply Sequence (per device)
|
||||
# 1. Validate the address shape
|
||||
# 2. Absent from sysfs → already done, skip silently
|
||||
# 3. Bound to a driver other than vfio-pci → refuse, warn, continue
|
||||
# 4. Claimed by a running VM → refuse, warn, continue
|
||||
# 5. Write 1 to the device's sysfs remove node
|
||||
# 6. Verify it left the bus
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Required
|
||||
# Writing to /sys/bus/pci/devices requires root.
|
||||
#
|
||||
# Driver Guard
|
||||
# Only vfio-pci-bound or unbound devices are eligible. This is what stops a
|
||||
# mistyped address from pulling the SAS controller, a NIC or an NVMe drive out
|
||||
# from under a running system.
|
||||
#
|
||||
# Running VM Guard
|
||||
# A vfio-pci device may be passed through to a live VM. Removing it would rip
|
||||
# the device away mid-flight. Running domains are checked for the address.
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs at array start.
|
||||
#
|
||||
# Disabled By Default
|
||||
# PCIE_QUIET_ENABLED is false and the device list is empty until an operator
|
||||
# fills it in per host. An empty list exits silently, so a host that has no
|
||||
# such hardware runs a no-op.
|
||||
#
|
||||
# Silent on Success
|
||||
# Runs every boot — no noise when the devices are already gone.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PCIE_QUIET_ENABLED
|
||||
# Master gate. Nothing is removed while false. (default: false)
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_PCIE_QUIET_DEVICES
|
||||
# PCI addresses to remove, full DDDD:BB:DD.F form. (default: empty)
|
||||
# Aliased to PCIE_QUIET_DEVICES by detect_hosts.
|
||||
#
|
||||
# HOST1 example — both are onboard, stubbed to vfio, used by nothing:
|
||||
# "0000:03:00.0" ASMedia ASM1143 USB 3.1 controller
|
||||
# "0000:04:00.0" Intel Wireless 8265
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# pcie_aer_quiet.sh
|
||||
# Remove every eligible listed device. Silent if all are already absent.
|
||||
#
|
||||
# pcie_aer_quiet.sh --dry-run
|
||||
# Show what would be removed, refused or skipped. No sysfs write.
|
||||
#
|
||||
# pcie_aer_quiet.sh --status
|
||||
# Show each listed device, its driver, and whether it is present.
|
||||
#
|
||||
# pcie_aer_quiet.sh --log
|
||||
# Verbose output showing each guard decision.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — removing PCI devices requires root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no devices will be removed"
|
||||
|
||||
PCI_DEVICE_ROOT="/sys/bus/pci/devices"
|
||||
PCI_ADDR_PATTERN='^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]$'
|
||||
|
||||
# ── Which driver currently owns the device, or "none" ────────────────────────────────────────
|
||||
pcie_driver_of() {
|
||||
local addr="$1" link
|
||||
link=$(readlink "$PCI_DEVICE_ROOT/$addr/driver" 2>/dev/null) || { echo "none"; return 0; }
|
||||
[[ -n "$link" ]] && basename "$link" || echo "none"
|
||||
}
|
||||
|
||||
# ── Is the address passed through to a VM that is running right now? ─────────────────────────
|
||||
# Returns the domain name on stdout when claimed, empty when free.
|
||||
pcie_claimed_by_vm() {
|
||||
local addr="$1"
|
||||
command -v virsh >/dev/null 2>&1 || return 0
|
||||
|
||||
local domain bus slot func xml_addr dom
|
||||
domain="${addr%%:*}"
|
||||
bus="${addr#*:}"; bus="${bus%%:*}"
|
||||
slot="${addr##*:}"; slot="${slot%%.*}"
|
||||
func="${addr##*.}"
|
||||
|
||||
xml_addr="domain='0x${domain}' bus='0x${bus}' slot='0x${slot}' function='0x${func}'"
|
||||
|
||||
while read -r dom; do
|
||||
[[ -z "$dom" ]] && continue
|
||||
if virsh dumpxml "$dom" 2>/dev/null | awk '/<hostdev/,/<\/hostdev>/' | \
|
||||
grep -qF "$xml_addr"; then
|
||||
echo "$dom"
|
||||
return 0
|
||||
fi
|
||||
done < <(virsh list --name --state-running 2>/dev/null)
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Enabled: $PCIE_QUIET_ENABLED"
|
||||
echo "$ICON_PLUGIN Devices: ${#PCIE_QUIET_DEVICES[@]} listed"
|
||||
echo ""
|
||||
|
||||
if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
|
||||
echo " $ICON_SKIP No devices configured — nothing to do on this host"
|
||||
else
|
||||
for addr in "${PCIE_QUIET_DEVICES[@]}"; do
|
||||
if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
|
||||
echo " $ICON_ERROR $addr — malformed address"
|
||||
elif [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
||||
echo " $ICON_SUCCESS $addr — already absent from the bus ✅"
|
||||
else
|
||||
echo " $ICON_WARN $addr — present, driver: $(pcie_driver_of "$addr")"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Gates ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$PCIE_QUIET_ENABLED" != true ]]; then
|
||||
log "$ICON_SKIP PCIE_QUIET_ENABLED is false — nothing removed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#PCIE_QUIET_DEVICES[@]} -eq 0 ]]; then
|
||||
log "$ICON_SKIP No PCIe devices configured for $MY_ID — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Remove ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
|
||||
REMOVED=0
|
||||
ALREADY=0
|
||||
REFUSED=0
|
||||
|
||||
for addr in "${PCIE_QUIET_DEVICES[@]}"; do
|
||||
|
||||
# ── Shape ────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ ! "$addr" =~ $PCI_ADDR_PATTERN ]]; then
|
||||
error "Malformed PCI address, refusing: '$addr' (expected DDDD:BB:DD.F)"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Present? ─────────────────────────────────────────────────────────────────────────────
|
||||
if [[ ! -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
||||
log "$ICON_SUCCESS $addr already absent"
|
||||
ALREADY=$(( ALREADY + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Driver guard ─────────────────────────────────────────────────────────────────────────
|
||||
DRIVER=$(pcie_driver_of "$addr")
|
||||
if [[ "$DRIVER" != "vfio-pci" && "$DRIVER" != "none" ]]; then
|
||||
error "$addr is bound to '$DRIVER' — refusing to remove a device in use"
|
||||
error "Only vfio-pci-bound or unbound devices are eligible. Stub it first, or remove it from PCIE_QUIET_DEVICES."
|
||||
notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — bound to $DRIVER" \
|
||||
"PCIe AER Quiet" "warning"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Running VM guard ─────────────────────────────────────────────────────────────────────
|
||||
CLAIMED_BY=$(pcie_claimed_by_vm "$addr")
|
||||
if [[ -n "$CLAIMED_BY" ]]; then
|
||||
error "$addr is passed through to running VM '$CLAIMED_BY' — refusing to remove"
|
||||
notify "PCIe quiet refused $addr on $(hostname) ($MY_ID) — in use by VM $CLAIMED_BY" \
|
||||
"PCIe AER Quiet" "warning"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Apply ────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove $addr (driver: $DRIVER)"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Removing $addr (driver: $DRIVER)..."
|
||||
if ! echo 1 > "$PCI_DEVICE_ROOT/$addr/remove" 2>/dev/null; then
|
||||
error "Write to $PCI_DEVICE_ROOT/$addr/remove failed"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── Verify ───────────────────────────────────────────────────────────────────────────────
|
||||
if [[ -e "$PCI_DEVICE_ROOT/$addr" ]]; then
|
||||
error "$addr still present after remove — kernel refused"
|
||||
REFUSED=$(( REFUSED + 1 ))
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Removed $addr ✅"
|
||||
REMOVED=$(( REMOVED + 1 ))
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
if [[ $REMOVED -eq 0 && $REFUSED -eq 0 ]]; then
|
||||
log "$ICON_SUCCESS All ${ALREADY} configured device(s) already absent"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PCIe AER QUIET SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SUCCESS Removed: $REMOVED"
|
||||
echo "$ICON_SKIP Already: $ALREADY"
|
||||
[[ $REFUSED -gt 0 ]] && echo "$ICON_ERROR Refused: $REFUSED"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
if [[ $REFUSED -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: completed with refusals ⚠️"
|
||||
else
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
Executable → Regular
+14
-14
@@ -18,6 +18,19 @@
|
||||
# total. Too high wastes RAM; too low causes slowdowns.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Read the current pm.max_children from the PHP-FPM pool config
|
||||
# 2. Already at or above the target → exit silently, no write, no restart
|
||||
# 3. Otherwise rewrite the value and restart PHP-FPM via the adapter
|
||||
# 4. Verify PHP-FPM came back up
|
||||
#
|
||||
# Runs at array start, before the WebGUI sees real load. The setting does not survive an
|
||||
# unRAID update — the OS replaces the pool config — which is why this reapplies every boot
|
||||
# rather than being a one-time install step.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -41,19 +54,6 @@
|
||||
# 7. Read back config to confirm value applied
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Read the current pm.max_children from the PHP-FPM pool config
|
||||
# 2. Already at or above the target → exit silently, no write, no restart
|
||||
# 3. Otherwise rewrite the value and restart PHP-FPM via the adapter
|
||||
# 4. Verify PHP-FPM came back up
|
||||
#
|
||||
# Runs at array start, before the WebGUI sees real load. The setting does not survive an
|
||||
# unRAID update — the OS replaces the pool config — which is why this reapplies every boot
|
||||
# rather than being a one-time install step.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -257,4 +257,4 @@ echo ""
|
||||
echo "$ICON_DONE Status: done ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
exit 0
|
||||
|
||||
Executable → Regular
+57
-23
@@ -14,6 +14,20 @@
|
||||
# page always reflects the live key value.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Check whether a Varaverk key already exists in the unraid-api registry
|
||||
# 2. Create or overwrite it — the registry is ephemeral, so re-registering is the norm
|
||||
# 3. Write the resulting key into this host's conf, replacing any previous value
|
||||
# 4. Push the key into each partner's OWN conf, at the path their varaverk.cfg reports
|
||||
# 5. Report whether the key was created, refreshed, or unchanged
|
||||
#
|
||||
# Runs at array start. The registry does not survive OS updates or an unraid-api restart,
|
||||
# which is why this re-registers unconditionally rather than only when the key is missing —
|
||||
# a key present in the conf but absent from the registry is the exact failure it repairs.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -28,19 +42,6 @@
|
||||
# without a separate sync step.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Check whether a Varaverk key already exists in the unraid-api registry
|
||||
# 2. Create or overwrite it — the registry is ephemeral, so re-registering is the norm
|
||||
# 3. Write the resulting key into this host's conf, replacing any previous value
|
||||
# 4. Report whether the key was created, refreshed, or unchanged
|
||||
#
|
||||
# Runs at array start. The registry does not survive OS updates or an unraid-api restart,
|
||||
# which is why this re-registers unconditionally rather than only when the key is missing —
|
||||
# a key present in the conf but absent from the registry is the exact failure it repairs.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -49,6 +50,17 @@
|
||||
# Conf file check — aborts before any writes if the host conf is missing
|
||||
# dry-run mode — shows what would happen without touching anything
|
||||
#
|
||||
# Remote path discovery
|
||||
# The partner's conf path comes from resolve_remote_scripts_dir(), which reads their
|
||||
# varaverk.cfg, so a partner in appdata storage mode is found. The path was hardcoded
|
||||
# to the flash plugin directory, which is wrong for any such partner.
|
||||
#
|
||||
# Remote target must exist
|
||||
# The pushed script refuses to create the conf and reports the path it looked at.
|
||||
# resolve_remote_scripts_dir() falls back to our own SCRIPTS_DIR when the probe fails,
|
||||
# and appending an API key to a merely plausible path is how the hardcoded version
|
||||
# failed without saying so.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
@@ -135,7 +147,14 @@ if [[ -n "$KEY" ]]; then
|
||||
PREVIEW="${KEY:0:8}...${KEY: -4}"
|
||||
# Always sync registry key → conf, even if the key was already there.
|
||||
# Conf gets wiped on git pull / conf regeneration without touching the registry.
|
||||
CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
|
||||
# Sourced, not pattern-matched. This was a grep -oP with a variable-length lookbehind, which
|
||||
# Unraid's grep is ugrep and rejects outright — "length of lookbehind assertion is not
|
||||
# limited", rc 2, swallowed by the || true. CONF_HAS_KEY was therefore always empty, the
|
||||
# comparison below never matched, and this script logged "conf is stale — syncing" and
|
||||
# rewrote the same key into the conf every fifteen minutes since it was written.
|
||||
# Sourcing also means the value is read the way bash reads it, escapes and all.
|
||||
CONF_HAS_KEY=$(bash -c 'source "$1" >/dev/null 2>&1 || exit 0; printf "%s" "${!2-}"' \
|
||||
_ "$CONF_FILE" "$VAR_NAME" 2>/dev/null || true)
|
||||
if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
|
||||
echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
|
||||
log "Key in registry and conf — no action needed"
|
||||
@@ -187,6 +206,9 @@ if [[ -z "$SSH_KEY" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# resolve_remote_scripts_dir() reads this; every inline timeout below already uses 10.
|
||||
SSH_TIMEOUT=10
|
||||
|
||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
|
||||
partner_host="${!host_var}"
|
||||
[[ -z "$partner_host" ]] && continue
|
||||
@@ -196,16 +218,27 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
|
||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||
[[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
|
||||
|
||||
# Target is the partner's OWN conf on their machine
|
||||
partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
|
||||
# Target is the partner's OWN conf on their machine. Their SCRIPTS_DIR is read from their
|
||||
# varaverk.cfg rather than assumed — this was hardcoded to the flash plugin path, so a
|
||||
# partner in appdata storage mode had its key appended under a directory that does not
|
||||
# exist there. HOST2 has run in appdata mode since it was installed.
|
||||
partner_sd=$(resolve_remote_scripts_dir "$partner_ip" "$SSH_KEY" "no")
|
||||
partner_conf="${partner_sd}/Configurations/${partner_slot}.conf"
|
||||
tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
|
||||
remote="/tmp/vv_kp_${RANDOM}.sh"
|
||||
chmod 700 "$tmp"
|
||||
|
||||
# Key stays in the temp file — never appears in SSH command args
|
||||
# Key stays in the temp file — never appears in SSH command args.
|
||||
# The conf must already exist: resolve_remote_scripts_dir falls back to our own SCRIPTS_DIR
|
||||
# when the probe fails, and appending a key to a path that is merely plausible is how the
|
||||
# hardcoded version failed silently. Report the path instead of guessing.
|
||||
cat > "$tmp" <<PUSHSCRIPT
|
||||
#!/bin/sh
|
||||
target='${partner_conf}'
|
||||
if [ ! -f "\$target" ]; then
|
||||
echo "missing:\$target"
|
||||
exit 1
|
||||
fi
|
||||
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
|
||||
sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
|
||||
else
|
||||
@@ -216,13 +249,14 @@ PUSHSCRIPT
|
||||
|
||||
if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
|
||||
if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
push_out=$(timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||
-o StrictHostKeyChecking=no "root@${partner_ip}" \
|
||||
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
|
||||
echo "Key pushed to $partner_host ✅"
|
||||
else
|
||||
warn "Key push to $partner_host failed — they can create their own copy"
|
||||
fi
|
||||
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null)
|
||||
case "$push_out" in
|
||||
*ok*) echo "Key pushed to $partner_host ✅" ;;
|
||||
missing:*) warn "Key push to $partner_host failed — no conf at ${push_out#missing:}" ;;
|
||||
*) warn "Key push to $partner_host failed — they can create their own copy" ;;
|
||||
esac
|
||||
else
|
||||
warn "SCP to $partner_host failed — skipping"
|
||||
fi
|
||||
|
||||
@@ -23,7 +23,7 @@ containers, VMs, transcodes, arr library counts. Building that live on every pag
|
||||
dozens of API calls and `docker inspect` runs per refresh, from a WebGUI that is already the
|
||||
first thing to slow down under load.
|
||||
|
||||
Instead, both are written to `/tmp/vv_cache` (tmpfs — RAM-speed reads, cleared on reboot) and
|
||||
Instead, both are written to `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs — RAM-speed reads, cleared on reboot) and
|
||||
the pages serve from there.
|
||||
|
||||
### ⚡ `api_cache_writer.sh` — local payloads, every minute
|
||||
@@ -128,7 +128,7 @@ that prevents shadow directories.
|
||||
│
|
||||
Tools/api_cache_writer.php ─────┘ same builders → cache can't disagree with live
|
||||
│
|
||||
└─► /tmp/vv_cache/monitor.json ──► served by api/monitor.php unless ?live=1
|
||||
└─► /tmp/varaverk/api/monitor.json ──► served by api/monitor.php unless ?live=1
|
||||
```
|
||||
|
||||
Adding a metric means adding it in `include/` once. The page, the live endpoint, and the cache
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+177
@@ -0,0 +1,177 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# =============================== AI Explain Check =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# Runs every case in ai_explain_fixtures.txt through the worker's --explain mode and checks the
|
||||
# routing it reports against what the fixture says it should be. Catches a guard that has been
|
||||
# undone by a later guard, which is the failure this subsystem keeps producing.
|
||||
#
|
||||
# OPERATIONAL MODEL
|
||||
# Never calls the model. --explain stops on the line where deterministic assembly ends, so a
|
||||
# full pass costs about a second per case and returns the same answer every time. There is no
|
||||
# Ollama dependency, no token spend, and no flaky wording to chase.
|
||||
#
|
||||
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
|
||||
# when the routing changes, not every night. Nothing on the running system depends on it.
|
||||
#
|
||||
# DESIGN PRINCIPLES
|
||||
# Asserts routing, never wording.
|
||||
# Which capabilities a profile holds and which evidence was attached are decided before the
|
||||
# model is asked anything. Asserting on generated prose would fail for reasons that tell
|
||||
# nobody anything and the check would be ignored within a fortnight.
|
||||
#
|
||||
# A malformed assertion fails loudly rather than passing quietly.
|
||||
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
|
||||
# is worse than no assertion, because the line still reads as covered.
|
||||
#
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# Read-only. Runs fixtures through --explain and compares; writes nothing and changes no state.
|
||||
#
|
||||
# Exits non-zero on any failure, so it can gate a commit.
|
||||
#
|
||||
# RUNTIME MODES
|
||||
# ai_explain_check.sh check every fixture
|
||||
# ai_explain_check.sh --verbose print the full explain report for each case
|
||||
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
|
||||
#
|
||||
# DEPENDS ON
|
||||
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
|
||||
# Plugin/unraid/Tools/ai_explain_fixtures.txt
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WORKER="$HERE/ai_chat_worker.php"
|
||||
FIXTURES="$HERE/ai_explain_fixtures.txt"
|
||||
|
||||
VERBOSE=false
|
||||
FILTER=""
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--verbose) VERBOSE=true ;;
|
||||
*) FILTER="$a" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -f "$WORKER" ]] || { echo "missing worker: $WORKER"; exit 2; }
|
||||
[[ -f "$FIXTURES" ]] || { echo "missing fixtures: $FIXTURES"; exit 2; }
|
||||
|
||||
PASS=0; FAIL=0; SKIP=0
|
||||
FAILED_LINES=()
|
||||
|
||||
trim() { local s="$1"; s="${s#"${s%%[![:space:]]*}"}"; s="${s%"${s##*[![:space:]]}"}"; printf '%s' "$s"; }
|
||||
|
||||
# One field out of the explain report. Everything it reads is a fixed label printed by --explain.
|
||||
field() {
|
||||
local report="$1" label="$2"
|
||||
printf '%s' "$report" | grep -m1 -E "^ *$label " | sed -E "s/^ *$label +//" | sed -E 's/ +$//'
|
||||
}
|
||||
|
||||
lineno=0
|
||||
while IFS= read -r raw || [[ -n "$raw" ]]; do
|
||||
lineno=$((lineno + 1))
|
||||
line="$(trim "$raw")"
|
||||
[[ -z "$line" || "$line" == \#* ]] && continue
|
||||
|
||||
IFS='|' read -r q prof scope kind expect <<< "$line"
|
||||
q="$(trim "$q")"; prof="$(trim "$prof")"; scope="$(trim "$scope")"
|
||||
kind="$(trim "$kind")"; expect="$(trim "$expect")"
|
||||
[[ -z "$q" ]] && continue
|
||||
|
||||
if [[ -n "$FILTER" && "$q" != *"$FILTER"* ]]; then SKIP=$((SKIP + 1)); continue; fi
|
||||
|
||||
report="$(php "$WORKER" --explain "$q" "$prof" "$scope" "$kind" 2>&1)"
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "✗ line $lineno: explain failed — $q"
|
||||
printf '%s\n' "$report" | head -3 | sed 's/^/ /'
|
||||
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno"); continue
|
||||
fi
|
||||
|
||||
# PROFILE prints "asked -> used (escalated)" on a handoff and just the profile otherwise.
|
||||
got_profile="$(printf '%s' "$report" | grep -m1 '^PROFILE' | sed -E 's/^PROFILE +//')"
|
||||
got_profile="${got_profile##*-> }"
|
||||
got_profile="$(printf '%s' "$got_profile" | sed -E 's/ *\(.*\)$//')"
|
||||
got_caps="$(printf '%s' "$report" | grep -m1 '^CAPS' | sed -E 's/^CAPS +//')"
|
||||
got_target="$(field "$report" 'named target')"
|
||||
got_run="$(field "$report" 'run outcome')"
|
||||
got_diag="$(field "$report" 'diagnostic')"
|
||||
got_attached="$(printf '%s' "$report" | sed -n '/^ATTACHED/,/^$/p' | tail -n +2 \
|
||||
| awk 'NF {print $1}' | tr '\n' ',' )"
|
||||
|
||||
problems=()
|
||||
for assert in $expect; do
|
||||
key="${assert%%=*}"; want="${assert#*=}"
|
||||
case "$key" in
|
||||
profile)
|
||||
[[ "$got_profile" == "$want" ]] || problems+=("profile: want $want, got $got_profile") ;;
|
||||
target)
|
||||
if [[ "$want" == "none" ]]; then
|
||||
[[ "$got_target" == "(none resolved)" ]] || problems+=("target: want none, got $got_target")
|
||||
else
|
||||
[[ "$got_target" == "$want" ]] || problems+=("target: want $want, got $got_target")
|
||||
fi ;;
|
||||
run)
|
||||
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
||||
[[ "$got_run" == "$exp" ]] || problems+=("run outcome: want $exp, got $got_run") ;;
|
||||
diag)
|
||||
exp=$([[ "$want" == "yes" ]] && echo YES || echo no)
|
||||
[[ "$got_diag" == "$exp" ]] || problems+=("diagnostic: want $exp, got $got_diag") ;;
|
||||
caps)
|
||||
# none holds nothing at all
|
||||
# only:a,b holds exactly these and nothing else
|
||||
# a holds at least this
|
||||
#
|
||||
# only: exists because General Chat's guarantee is not "it can search" but "search
|
||||
# is the only thing it can do". A presence check would still pass on the day
|
||||
# something else is granted there, which is the day the check was written for.
|
||||
if [[ "$want" == "none" ]]; then
|
||||
[[ "$got_caps" == "(none"* ]] || problems+=("caps: want none, got $got_caps")
|
||||
elif [[ "$want" == only:* ]]; then
|
||||
want_set="${want#only:}"
|
||||
got_set="$(printf '%s' "$got_caps" | tr -d ' ' | tr ',' '\n' | sort | paste -sd, -)"
|
||||
exp_set="$(printf '%s' "$want_set" | tr -d ' ' | tr ',' '\n' | sort | paste -sd, -)"
|
||||
[[ "$got_set" == "$exp_set" ]] \
|
||||
|| problems+=("caps: want exactly [$exp_set], got [$got_set]")
|
||||
else
|
||||
[[ "$got_caps" == *"$want"* ]] || problems+=("caps: want $want in [$got_caps]")
|
||||
fi ;;
|
||||
has)
|
||||
IFS=',' read -ra want_keys <<< "$want"
|
||||
for w in "${want_keys[@]}"; do
|
||||
[[ ",$got_attached" == *",$w,"* ]] || problems+=("missing attachment: $w")
|
||||
done ;;
|
||||
hasnt)
|
||||
IFS=',' read -ra bad_keys <<< "$want"
|
||||
for b in "${bad_keys[@]}"; do
|
||||
[[ ",$got_attached" == *",$b,"* ]] && problems+=("attached but must not be: $b")
|
||||
done ;;
|
||||
*)
|
||||
problems+=("unknown assertion '$key' — typo, or a key this checker does not know") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ${#problems[@]} -eq 0 ]]; then
|
||||
PASS=$((PASS + 1))
|
||||
printf '\033[32m✓\033[0m %-6s %s\n' "$prof" "$q"
|
||||
else
|
||||
FAIL=$((FAIL + 1)); FAILED_LINES+=("$lineno")
|
||||
printf '\033[31m✗\033[0m %-6s %s\n' "$prof" "$q"
|
||||
for p in "${problems[@]}"; do echo " $p"; done
|
||||
fi
|
||||
|
||||
[[ "$VERBOSE" == true ]] && printf '%s\n\n' "$report" | sed 's/^/ /'
|
||||
done < "$FIXTURES"
|
||||
|
||||
echo
|
||||
echo "─────────────────────────────────────────────"
|
||||
printf 'passed %d failed %d' "$PASS" "$FAIL"
|
||||
[[ $SKIP -gt 0 ]] && printf ' skipped %d' "$SKIP"
|
||||
echo
|
||||
if [[ $FAIL -gt 0 ]]; then
|
||||
echo "failing fixture lines: ${FAILED_LINES[*]}"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
@@ -0,0 +1,109 @@
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Expected routing for the AI assistant, one case per line, checked by ai_explain_check.sh.
|
||||
#
|
||||
# Every line here is a bug that was found by reading an answer and noticing it was wrong. That is
|
||||
# the expensive way to find them and it does not scale: the operator is the test suite, and the
|
||||
# operator is busy. Written down, each one costs a second to re-check forever.
|
||||
#
|
||||
# These assert ROUTING, never wording. What a profile is allowed, which script a question names,
|
||||
# which gates fire and what evidence gets attached are all decided before the model is involved,
|
||||
# so they are identical every run. The prose is not and is deliberately not asserted — a test that
|
||||
# depends on how the model phrases something fails for reasons nobody wants to read about.
|
||||
#
|
||||
# FORMAT
|
||||
# question | profile | scope | kind | assertions
|
||||
#
|
||||
# Blank fields are allowed. Assertions are space-separated:
|
||||
# profile=X the profile that ends up answering, after any handoff
|
||||
# target=X resolved run target, or 'none'
|
||||
# run=yes|no the run-outcome gate
|
||||
# diag=yes|no the diagnostic gate
|
||||
# caps=none the profile holds no capabilities at all
|
||||
# caps=only:a,b it holds exactly these and nothing else
|
||||
# caps=a it holds at least this
|
||||
# has=a,b every one of these must be attached
|
||||
# hasnt=a,b none of these may be attached
|
||||
#
|
||||
# Attachment keys: health warnings run_record log_tail log_missing incidents conf_keys
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# ── Run-outcome questions must arrive with the run, not with directions to the log panel ───────
|
||||
# The phrasing that started it. "went last" matched; "the run went" did not, one word order apart.
|
||||
lets check the daily orch log and see how the run went | varaverk | Scheduler | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
|
||||
lets look at daily orch log and see how it went last run | varaverk | Scheduler | | run=yes has=run_record,log_tail
|
||||
# Confirmed good in the WebGUI dock. Plural "logs", and "how it went" rather than "how the run
|
||||
# went" — neither exact string was covered, and both are how the question actually gets typed.
|
||||
lets check the daily orch logs and see how it went | varaverk | Scheduler | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
|
||||
how did the daily orch go | varaverk | | | target=Orchestrators/daily_sync_maintenance run=yes has=run_record,log_tail
|
||||
how did the weekly orch go last run | varaverk | | | target=Orchestrators/weekly_sync_maintenance run=yes has=run_record
|
||||
did the watchdog orchestrator run | varaverk | | | target=Orchestrators/watchdog_orchestrator run=yes
|
||||
give me a rundown of the daily orch | varaverk | | | run=yes has=run_record
|
||||
how long did the daily orch take | varaverk | | | run=yes has=run_record
|
||||
|
||||
# Works from the AI tab, which sends no scope at all — resolution is from the question, not the page.
|
||||
how did the daily orch go last run | varaverk | | | target=Orchestrators/daily_sync_maintenance has=run_record
|
||||
|
||||
# ── A clean run earns the record and the log, and nothing else ─────────────────────────────────
|
||||
# The health sweep and the recent-warning lines are ~660 tokens that answer nothing when the run
|
||||
# exited 0, taken straight out of the passages' budget. They come back the moment it did not.
|
||||
lets check the daily orch logs and see how it went | varaverk | Scheduler | | diag=no hasnt=health,warnings has=run_record,log_tail
|
||||
how did the daily orch go | varaverk | | | diag=no hasnt=health,warnings
|
||||
why did the daily orch fail | varaverk | | | diag=yes has=health,warnings,log_tail
|
||||
why did the weekly orch fail | varaverk | | | diag=yes has=health,warnings
|
||||
|
||||
# ── Definitional questions must NOT be answered with last night's log ──────────────────────────
|
||||
what does the daily orchestrator do | varaverk | | | target=Orchestrators/daily_sync_maintenance run=no hasnt=run_record,log_tail
|
||||
how does the daily orch work | varaverk | | | run=no hasnt=run_record,log_tail
|
||||
what is the daily orchestrator | varaverk | | | run=no hasnt=run_record
|
||||
|
||||
# ── Ambiguity resolves to nothing rather than guessing a script ────────────────────────────────
|
||||
# A scored match here attaches the wrong log and answers confidently about a run nobody asked
|
||||
# about, which is indistinguishable from a right answer unless you already knew.
|
||||
how did sync go | varaverk | | | target=none run=no
|
||||
what does RSYNC_ENABLED do | varaverk | | | target=none run=no has=conf_keys
|
||||
# Whole words only. "fail" contains "ai", and ai.log is a real log id — a substring match here
|
||||
# attached the AI's own log to questions about everything else.
|
||||
why did this run fail | varaverk | | | target=none
|
||||
is the mail relay working again | varaverk | | | target=none
|
||||
|
||||
# ── The Recent Activity buttons: one click from a row to an answer about that run ──────────────
|
||||
# Both open the log first, so the dock is scoped to troubleshoot against that script's log id.
|
||||
Why did this run fail? | troubleshoot | Orchestrators/daily_sync_maintenance | | diag=yes has=log_tail,run_record
|
||||
How did this run go? | troubleshoot | Orchestrators/weekly_sync_maintenance | | has=log_tail,run_record
|
||||
|
||||
# ── General Chat holds nothing, and hands Varaverk questions up rather than deferring ──────────
|
||||
how was your day | chat | | | profile=chat caps=only:web_search hasnt=health,log_tail,incidents,conf_keys
|
||||
what does arr_sync.sh do | chat | | | profile=varaverk
|
||||
is RSYNC_ENABLED on right now | chat | | | profile=varaverk has=conf_keys
|
||||
how did the daily orch go | chat | | | profile=varaverk run=yes has=run_record
|
||||
|
||||
# Chat opened against a script must not be handed the operator's own incident notes about it.
|
||||
how was your day | chat | Orchestrators/daily_sync_maintenance | | profile=chat hasnt=incidents
|
||||
|
||||
# ── Troubleshoot gets the log for whatever is open, plus the right to file a bug ───────────────
|
||||
why did the weekly orch fail | troubleshoot | Orchestrators/weekly_sync_maintenance | | diag=yes has=log_tail,run_record
|
||||
what is going on here | troubleshoot | Orchestrators/daily_sync_maintenance | | diag=yes has=log_tail
|
||||
|
||||
# ── The code profile answers from the model alone: no passages, no live state ──────────────────
|
||||
write me a script that copies a folder | code | | | hasnt=health,log_tail,incidents,conf_keys
|
||||
|
||||
# ── General Chat routes to the profile that fits, not always to the Assistant ──────────────────
|
||||
# Chat had one escalation target, so a fault report and a request for code both landed on the
|
||||
# documentation assistant. Both were answered adequately; neither was answered well. Order is the
|
||||
# policy — code is checked first because it is the only intent about a thing that does not exist
|
||||
# yet, and troubleshoot requires diagnostic phrasing AND something here to diagnose.
|
||||
why did the daily orch fail | chat | | | profile=troubleshoot diag=yes target=Orchestrators/daily_sync_maintenance
|
||||
the daily orch is broken | chat | | | profile=troubleshoot diag=yes
|
||||
why did arr_sync.sh error out | chat | | | profile=troubleshoot diag=yes
|
||||
create a script to prune old logs | chat | | | profile=code hasnt=health,log_tail,incidents
|
||||
make me a one-liner that counts files | chat | | | profile=code
|
||||
write me a bash script for backups | chat | | | profile=code
|
||||
|
||||
# Diagnostic phrasing about nothing here stays in chat — "why" is not a Varaverk question on its
|
||||
# own, and escalating adds capability, so a wrong escalation costs more than a missed one.
|
||||
why is the sky blue | chat | | | profile=chat caps=only:web_search hasnt=health,log_tail,incidents,conf_keys
|
||||
what is wrong with my car | chat | | | profile=chat caps=only:web_search
|
||||
|
||||
# Naming a script and asking what it does is documentation, not a request to write one. The code
|
||||
# router is anchored on the verb for exactly this pair.
|
||||
what does arr_sync.sh do | chat | | | profile=varaverk
|
||||
@@ -0,0 +1,155 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Recall and precision for the syslog triage patterns. See ai_log_check.sh for why both
|
||||
// halves exist and why neither alone is enough.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing,
|
||||
// and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Only recall can fail the run.
|
||||
// A missed fixture is a fact about the patterns and is always a defect. A precision hit is
|
||||
// a fact about this machine — a genuinely failing disk should not turn this red, and if it
|
||||
// did, the honest fix would be to stop having a failing disk rather than to edit a pattern.
|
||||
//
|
||||
// Precision is replayed against real history, not a sample.
|
||||
// The patterns that cause damage are the ones matching ordinary operation, and ordinary
|
||||
// operation is exactly what a hand-written fixture file never contains. Only the machine's
|
||||
// own syslog can show what a pattern fires on when nothing is wrong.
|
||||
//
|
||||
// The sweep is never invoked, only its matcher.
|
||||
// vv_ai_syslog_findings() is called on lines this file supplies. Running the real sweep
|
||||
// would file findings, and a test that has to be cleaned up afterwards stops being run.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Reads ai_log_fixtures.txt and this host's /var/log/syslog*; files no finding,
|
||||
// writes no store, and touches no conf beyond the enable flag.
|
||||
//
|
||||
// Exit 0 when every fixture is recognised as written. Precision findings are reported but
|
||||
// never fail the run — see DESIGN PRINCIPLES.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php ai_log_check.php both checks
|
||||
// php ai_log_check.php --recall fixtures only
|
||||
// php ai_log_check.php --precision replay this host's syslog history only
|
||||
//
|
||||
// Not scheduled, and deliberately so. Run it after touching VV_AI_SYSLOG_PATTERNS.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||
|
||||
$args = array_slice($argv ?? [], 1);
|
||||
$only = in_array('--precision', $args, true) ? 'precision'
|
||||
: (in_array('--recall', $args, true) ? 'recall' : 'both');
|
||||
$fixtures = __DIR__ . '/ai_log_fixtures.txt';
|
||||
|
||||
$pass = 0; $fail = 0;
|
||||
function ok(string $what, bool $cond, string $got = ''): void {
|
||||
global $pass, $fail;
|
||||
if ($cond) { $pass++; printf(" ok %s\n", $what); }
|
||||
else { $fail++; printf(" FAIL %s%s\n", $what, $got !== '' ? "\n → $got" : ''); }
|
||||
}
|
||||
|
||||
// One line at a time, so a fixture is asserted on its own rather than on whatever aggregated
|
||||
// with it. The finders are given the line directly, bypassing the file, the timestamp filter and
|
||||
// docker — those are tested separately, and a fixture dated last August would otherwise be
|
||||
// silently dropped for being older than the marker.
|
||||
function classify(string $line, string $source = 'sys'): ?array {
|
||||
if ($source === 'ctr') {
|
||||
$f = vv_ai_container_findings(0, ['fixture-container' => [$line]]);
|
||||
return $f[0] ?? null;
|
||||
}
|
||||
$f = vv_ai_syslog_findings(0, [$line]);
|
||||
return $f[0] ?? null;
|
||||
}
|
||||
|
||||
if ($only !== 'precision') {
|
||||
echo "── recall: fixtures ──────────────────────────────────────────────\n";
|
||||
if (!is_readable($fixtures)) {
|
||||
echo " FAIL cannot read $fixtures\n";
|
||||
exit(1);
|
||||
}
|
||||
foreach (file($fixtures, FILE_IGNORE_NEW_LINES) as $n => $raw) {
|
||||
$line = rtrim($raw);
|
||||
if ($line === '' || $line[0] === '#') continue;
|
||||
|
||||
// A ! line must match nothing at all. ! source | line
|
||||
if ($line[0] === '!') {
|
||||
$rest = trim(substr($line, 1));
|
||||
$bits = explode('|', $rest, 2);
|
||||
if (count($bits) !== 2) { ok(sprintf('L%d is malformed', $n + 1), false, $line); continue; }
|
||||
$src = trim($bits[0]);
|
||||
$sample = trim($bits[1]);
|
||||
$got = classify($sample, $src);
|
||||
ok(sprintf('L%-3d %s no match: %s', $n + 1, $src, mb_substr($sample, 0, 58)),
|
||||
$got === null, $got ? "matched as {$got['ref']} / {$got['subject']}" : '');
|
||||
continue;
|
||||
}
|
||||
|
||||
$parts = explode('|', $line, 4);
|
||||
if (count($parts) !== 4) { ok(sprintf('L%d is malformed', $n + 1), false, $line); continue; }
|
||||
[$src, $wantSubject, $wantLevel, $sample] = array_map('trim', $parts);
|
||||
|
||||
$got = classify($sample, $src);
|
||||
if ($got === null) {
|
||||
ok(sprintf('L%-3d %s', $n + 1, mb_substr($sample, 0, 64)), false, 'no pattern matched');
|
||||
continue;
|
||||
}
|
||||
ok(sprintf('L%-3d %-12s %-5s %s', $n + 1, $got['subject'], $got['sys_level'],
|
||||
mb_substr($got['ref'], 0, 30)),
|
||||
$got['subject'] === $wantSubject && $got['sys_level'] === $wantLevel,
|
||||
sprintf('wanted %s/%s, got %s/%s', $wantSubject, $wantLevel,
|
||||
$got['subject'], $got['sys_level']));
|
||||
}
|
||||
}
|
||||
|
||||
if ($only !== 'recall') {
|
||||
echo "\n── precision: this host's real syslog history ────────────────────\n";
|
||||
$total = 0; $hits = [];
|
||||
foreach (glob('/var/log/syslog*') ?: [] as $file) {
|
||||
if (!is_readable($file)) continue;
|
||||
$fh = @fopen($file, 'r');
|
||||
if (!$fh) continue;
|
||||
while (($l = fgets($fh)) !== false) {
|
||||
$total++;
|
||||
$got = classify(rtrim($l));
|
||||
if ($got === null) continue;
|
||||
$k = $got['ref'] . ' | ' . $got['subject'] . ' | ' . $got['sys_level'];
|
||||
$hits[$k] = ($hits[$k] ?? 0) + 1;
|
||||
}
|
||||
fclose($fh);
|
||||
}
|
||||
printf(" scanned %s lines\n", number_format($total));
|
||||
if ($hits) { arsort($hits); foreach ($hits as $k => $n) printf(" %-7s %s\n", number_format($n), $k); }
|
||||
else { echo " nothing matched\n"; }
|
||||
|
||||
// Every running container, read far deeper than a sweep ever does. A pattern that is quiet
|
||||
// over this much real output is a pattern that will be quiet in service.
|
||||
echo "\n── precision: every running container's log ──────────────────────\n";
|
||||
$cTotal = 0; $hits = [];
|
||||
$names = vv_ai_running_containers();
|
||||
foreach ($names as $name) {
|
||||
$lines = vv_ai_container_log($name, 1, 2000);
|
||||
$cTotal += count($lines);
|
||||
$found = vv_ai_container_findings(0, [$name => $lines]);
|
||||
foreach ($found as $f) {
|
||||
$k = $f['ref'] . ' | ' . $f['subject'] . ' | ' . $f['sys_level'];
|
||||
$hits[$k] = ($hits[$k] ?? 0) + (int)filter_var($f['observed'], FILTER_SANITIZE_NUMBER_INT);
|
||||
}
|
||||
}
|
||||
printf(" scanned %s lines across %d container%s\n", number_format($cTotal), count($names),
|
||||
count($names) === 1 ? '' : 's');
|
||||
if (!$hits) {
|
||||
echo " nothing matched — this host is not reporting any of these faults\n";
|
||||
} else {
|
||||
arsort($hits);
|
||||
foreach ($hits as $k => $n) printf(" %-7s %s\n", number_format($n), $k);
|
||||
echo "\n Each line above is a fault the sweep would file. If any of them is normal\n"
|
||||
. " operation on this machine, the pattern is wrong — add it to the fixtures as a\n"
|
||||
. " ! line and tighten the pattern until it stops matching.\n";
|
||||
}
|
||||
}
|
||||
|
||||
printf("\n%d passed, %d failed\n", $pass, $fail);
|
||||
exit($fail ? 1 : 0);
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================== AI Log Pattern Check =======================================
|
||||
# ==============================================================================================
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Checks the repair sweep's syslog triage two ways, because the two failure modes are opposite
|
||||
# and a single test catches only one of them:
|
||||
#
|
||||
# RECALL — every line in ai_log_fixtures.txt is recognised, with the right subject and
|
||||
# the right level. Faults this host has never had, which is most of them.
|
||||
# PRECISION — replays this machine's entire real syslog history and reports everything the
|
||||
# patterns fire on. A pattern that matches normal operation fills the findings
|
||||
# store with noise and teaches the operator to ignore the notification.
|
||||
#
|
||||
# Run it after touching VV_AI_SYSLOG_PATTERNS. Nothing here writes: no findings are filed, no
|
||||
# conf is read for anything but the enable flag, and the sweep is never invoked.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in ai_log_check.php, next to the sweep's own matcher — the patterns and
|
||||
# vv_ai_syslog_findings() live in include/ai_repair.php, and a bash reimplementation of the
|
||||
# matching would be a second set of regexes to keep in step with the first.
|
||||
#
|
||||
# Flags are forwarded verbatim; nothing is interpreted here.
|
||||
#
|
||||
# Not scheduled and in no orchestrator. This is a development check that runs when the patterns
|
||||
# change, not on a timer — nothing on the running system depends on it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Two checks, because the failure modes are opposite.
|
||||
# Recall catches a pattern that stopped matching; precision catches one that matches too much.
|
||||
# A single test would catch one and silently permit the other, and the second is the one that
|
||||
# fills the findings store with noise until the operator stops reading it.
|
||||
#
|
||||
# Precision is measured against this machine's real history.
|
||||
# A hand-written fixture file can show that a pattern matches what it should. Only a real
|
||||
# syslog can show what it also matches when nothing is wrong.
|
||||
#
|
||||
# Only recall fails the run.
|
||||
# What a real syslog contains is a fact about the machine, not about the patterns. A genuinely
|
||||
# failing disk should not turn this red.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only. No finding is filed, no store is written, and the repair sweep itself is never run
|
||||
# — only its matcher, on lines this check supplies.
|
||||
#
|
||||
# Safe to run on a live host at any time, including one that is currently faulting. It observes
|
||||
# the syslog it replays and changes nothing about it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ai_log_check.sh both checks
|
||||
# ai_log_check.sh --precision replay the host's syslogs only, and list what matched
|
||||
# ai_log_check.sh --recall fixtures only
|
||||
#
|
||||
# ==============================================================================================
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/ai_log_check.php" "$@"
|
||||
@@ -0,0 +1,100 @@
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# Log lines the repair sweep must recognise, one case per line, checked by ai_log_check.sh.
|
||||
#
|
||||
# WHY THIS FILE EXISTS
|
||||
# The triage patterns for Varaverk's own logs were written against its emit sites — the format
|
||||
# strings are in this repo and can be read. The kernel's are not, and this host is healthy:
|
||||
# 74,519 lines of its real syslog contain PCIe AER errors and nothing else. So the patterns for
|
||||
# a failing disk or a corrupt filesystem cannot be verified against anything this machine has
|
||||
# said, and a pattern nobody has ever seen match is a pattern that does not work.
|
||||
#
|
||||
# These lines are what those faults look like. Precision is checked separately and against
|
||||
# reality — ai_syslog_check.sh replays the host's whole syslog history and fails if anything
|
||||
# fires that should not. This file is the other half: recall, on faults the host has not had.
|
||||
#
|
||||
# THE RULE FOR ADDING ONE
|
||||
# Paste the real line, from this machine or from a kernel that emitted it. Do not compose one
|
||||
# from what the pattern already matches — that tests the regex against itself and passes
|
||||
# forever while the real format drifts away underneath it.
|
||||
#
|
||||
# FORMAT
|
||||
# source | subject | level | line
|
||||
#
|
||||
# source sys = /var/log/syslog | ctr = a container log line
|
||||
#
|
||||
# subject what the finding must be about, after extraction
|
||||
# level warn | error
|
||||
# line the syslog line, verbatim, including its timestamp and host prefix
|
||||
#
|
||||
# A line beginning with ! must match NOTHING. Those are the near misses — normal operation that
|
||||
# reads like a fault — and they are the reason the pattern list is not simply /error/i.
|
||||
# Their format is: ! source | line
|
||||
# ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
# ── PCIe. Verified: this host emitted 427 of these across three syslogs. ──────────────────────
|
||||
sys | 0000:03:00.0 | warn | Aug 9 21:46:07 unRAID-Gmer4Lfe kernel: pcieport 0000:00:01.1: AER: Multiple Correctable error message received from 0000:03:00.0
|
||||
sys | 0000:02:02.0 | warn | Aug 7 11:02:41 unRAID-Gmer4Lfe kernel: pcieport 0000:00:03.1: AER: Correctable error message received from 0000:02:02.0
|
||||
sys | 0000:0a:00.0 | error | Aug 9 03:14:02 tower kernel: pcieport 0000:00:1c.0: AER: Uncorrected (Non-Fatal) error received from 0000:0a:00.0
|
||||
sys | 0000:0b:00.0 | error | Aug 9 03:14:03 tower kernel: pcieport 0000:00:1c.4: AER: Fatal error received from 0000:0b:00.0
|
||||
|
||||
# ── Block layer. The device is the subject; the ATA port above it is not actionable. ─────────
|
||||
sys | sdo | error | Aug 9 04:21:09 tower kernel: blk_update_request: critical medium error, dev sdo, sector 1953525161 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 0
|
||||
sys | sde | error | Aug 9 04:22:11 tower kernel: blk_update_request: I/O error, dev sde, sector 8 op 0x1:(WRITE) flags 0x800 phys_seg 0 prio class 0
|
||||
sys | sdc | error | Aug 9 04:23:00 tower kernel: Buffer I/O error on dev sdc, logical block 0, async page read
|
||||
sys | sdo | error | Aug 9 04:24:55 tower kernel: sd 1:0:3:0: [sdo] tag#28 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE unrecovered read error
|
||||
|
||||
# ── Filesystems. Each names its own device inside the parentheses. ────────────────────────────
|
||||
sys | md1 | error | Aug 9 05:00:01 tower kernel: XFS (md1): Metadata corruption detected at xfs_agf_verify+0x1f0/0x1f9 [xfs], xfs_agf block 0x1
|
||||
sys | md2 | error | Aug 9 05:01:02 tower kernel: XFS (md2): Internal error xfs_trans_cancel at line 990 of file fs/xfs/xfs_trans.c
|
||||
sys | sdb1 | error | Aug 9 05:02:03 tower kernel: BTRFS error (device sdb1): parent transid verify failed on 1298432 wanted 12 found 11
|
||||
sys | sdd1 | error | Aug 9 05:03:04 tower kernel: EXT4-fs error (device sdd1): ext4_find_entry:1455: inode #2: comm ls: reading directory lblock 0
|
||||
sys | filesystem | error | Aug 9 05:04:05 tower kernel: EXT4-fs (sdd1): Remounting filesystem read-only
|
||||
|
||||
# ── Memory and the kernel itself. ─────────────────────────────────────────────────────────────
|
||||
sys | shfs | error | Aug 9 06:00:00 tower kernel: Out of memory: Killed process 9823 (shfs) total-vm:2451234kB, anon-rss:1923412kB
|
||||
sys | mono | error | Aug 9 06:00:01 tower kernel: Out of memory: Killed process 1122 (mono) total-vm:900000kB, anon-rss:800000kB
|
||||
sys | kernel | error | Aug 9 07:00:00 tower kernel: kernel BUG at fs/xfs/xfs_message.c:102!
|
||||
sys | kernel | error | Aug 9 07:00:01 tower kernel: general protection fault: 0000 [#1] SMP NOPTI
|
||||
|
||||
# ── Near misses. Normal operation that reads like a fault; none of these may match. ───────────
|
||||
! sys | Aug 9 04:40:01 unRAID-Gmer4Lfe kernel: md: recovery thread: exit status: 0
|
||||
! sys | Aug 9 04:40:02 unRAID-Gmer4Lfe kernel: md: sync done. time=17452sec
|
||||
! sys | Aug 9 04:40:03 unRAID-Gmer4Lfe kernel: md: import disk6: (sdi) TOSHIBA_MG07ACA12TEY_Z0N0ANF9SG size: 11718885324
|
||||
! sys | Aug 9 12:00:00 unRAID-Gmer4Lfe shfs: /usr/sbin/zfs destroy 'cache/isos' 2>&1
|
||||
! sys | Aug 9 12:00:01 unRAID-Gmer4Lfe emhttpd: action: Enable all features using 'zpool upgrade'.
|
||||
! sys | Aug 9 12:00:02 unRAID-Gmer4Lfe root: Fix Common Problems: Error: Docker image file is getting full
|
||||
! sys | Aug 9 12:00:03 unRAID-Gmer4Lfe kernel: XFS (md1): Mounting V5 Filesystem
|
||||
! sys | Aug 9 12:00:04 unRAID-Gmer4Lfe kernel: XFS (md1): Ending clean mount
|
||||
! sys | Aug 9 12:00:05 unRAID-Gmer4Lfe sshd[1234]: error: kex_exchange_identification: Connection closed by remote host
|
||||
! sys | Aug 9 12:00:06 unRAID-Gmer4Lfe nginx: 2026/08/09 12:00:06 [error] 1#1: *1 open() failed
|
||||
! sys | Aug 9 12:00:07 unRAID-Gmer4Lfe kernel: docker0: port 3(veth1a2b3c) entered disabled state
|
||||
! sys | Aug 9 12:00:08 unRAID-Gmer4Lfe kernel: eth0: Link is Down
|
||||
|
||||
# ══ CONTAINER LOGS ═════════════════════════════════════════════════════════════════════════════
|
||||
# Fifty containers run here and they are fifty applications with nothing in common. These patterns
|
||||
# are about the environment underneath them — the strings come from libc, the kernel and SQLite,
|
||||
# so they are identical in every one. Precision was checked against all fifty containers' real
|
||||
# logs: 79,180 lines, zero matches, which is what a healthy machine should produce.
|
||||
#
|
||||
# The subject of a container finding is the container, so it is not asserted per line here — the
|
||||
# name comes from which log the line was read from, not from the text. Subject is left as the
|
||||
# container name the checker supplies.
|
||||
|
||||
ctr | fixture-container | error | 2026-08-09 12:00:00 ERROR sqlite3.DatabaseError: database disk image is malformed
|
||||
ctr | fixture-container | error | [Errno 28] No space left on device: '/config/logs/sonarr.txt'
|
||||
ctr | fixture-container | error | OSError: [Errno 30] Read-only file system: '/data/media'
|
||||
ctr | fixture-container | error | sqlite3.OperationalError: database or disk is full
|
||||
ctr | fixture-container | error | Error: EMFILE: too many open files, open '/config/db'
|
||||
ctr | fixture-container | error | curl: (60) SSL certificate problem: certificate has expired
|
||||
ctr | fixture-container | error | ssl.SSLCertVerificationError: certificate verify failed: unable to get local issuer
|
||||
ctr | fixture-container | error | write /var/lib/data: disk quota exceeded
|
||||
ctr | fixture-container | error | sqlite3.DatabaseError: file is not a database
|
||||
|
||||
# Near misses from real arr and media-server logs. None of these may match.
|
||||
! ctr | 2026-08-09 12:00:00 WARN sqlite3.OperationalError: database is locked, retrying in 200ms
|
||||
! ctr | 2026-08-09 12:00:01 INFO Import failed: file already exists in the destination
|
||||
! ctr | 2026-08-09 12:00:02 ERROR Permission denied reading /downloads/incomplete/foo.part
|
||||
! ctr | 2026-08-09 12:00:03 WARN Certificate expires in 21 days, renewal scheduled
|
||||
! ctr | 2026-08-09 12:00:04 ERROR Connection refused connecting to 127.0.0.1:9117
|
||||
! ctr | 2026-08-09 12:00:05 INFO Deleted 4 files to free space on device sda
|
||||
! ctr | 2026-08-09 12:00:06 ERROR HTTP 500 while calling /api/v3/command
|
||||
@@ -0,0 +1,121 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// One pass of the repair sweep. Reads the logs of jobs that finished since the last pass,
|
||||
// turns known error shapes into findings, probes for a correction, and either writes a proven
|
||||
// value or leaves the finding for the operator.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Called from ai_repair_sweep.sh, which the watchdog orchestrator runs. Everything it does
|
||||
// lives in include/ai_repair.php so the same code path is what the tests exercise; this file
|
||||
// owns only the lock, the log line, and the exit code.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Non-fatal, always.
|
||||
// Exits 0 on a disabled feature, a held lock, or a sweep that found nothing. Repair is an
|
||||
// enhancement — the orchestrator that runs it has real work to do either side, and this
|
||||
// must never be the reason a watchdog cycle reports failure.
|
||||
//
|
||||
// The summary is logged, not the reasoning.
|
||||
// What was scanned, what was found, what was written. The findings themselves are the
|
||||
// record; duplicating their contents into a log would be two copies to keep in step.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// One sweep at a time.
|
||||
// flock, non-blocking. A pass that overruns its fifteen-minute slot must not have a second
|
||||
// copy start probing and writing conf underneath it.
|
||||
//
|
||||
// Nothing is written unless two switches say so.
|
||||
// AI_REPAIR_ENABLED gates the sweep; AI_REPAIR_AUTOFIX_ENABLED gates writing. With only
|
||||
// the first, this reads and files and changes no configuration at all.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// ai_repair_sweep.php one pass
|
||||
// ai_repair_sweep.php --dry-run probe and report, write nothing, leave the marker alone
|
||||
// ai_repair_sweep.php --status what the last pass did, and what is open
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||
|
||||
$args = array_slice($argv ?? [], 1);
|
||||
$dryRun = in_array('--dry-run', $args, true);
|
||||
$status = in_array('--status', $args, true);
|
||||
|
||||
function rlog(string $msg): void {
|
||||
if (!is_dir(LOG_DIR)) return;
|
||||
@file_put_contents(LOG_DIR . '/ai_repair.log',
|
||||
date('Y-m-d H:i:s') . ' ' . $msg . "\n", FILE_APPEND | LOCK_EX);
|
||||
}
|
||||
|
||||
if ($status) {
|
||||
$last = vv_ai_sweep_last();
|
||||
$open = vv_ai_findings_list(['open', 'needs_operator']);
|
||||
printf("repair: %s\n", vv_ai_repair_enabled() ? 'enabled' : 'disabled');
|
||||
printf("autofix: %s\n", vv_ai_repair_autofix_enabled() ? 'enabled' : 'disabled (detect only)');
|
||||
printf("last pass: %s\n", $last ? date('Y-m-d H:i:s', $last) : 'never');
|
||||
printf("open findings: %d\n", count($open));
|
||||
// 'ref' rather than 'conf_key' — it is the conf key for the kinds that have one, and the
|
||||
// thing that identifies the finding for the kinds that do not. Printing the key left every
|
||||
// arr health row with a blank column.
|
||||
foreach ($open as $f) {
|
||||
printf(" %-7s %-9s %-28s seen %-4d %-14s %s\n",
|
||||
'[' . ($f['severity'] ?? '?') . ']',
|
||||
$f['subject'] ?? '?',
|
||||
$f['ref'] ?? ($f['conf_key'] ?? '?'),
|
||||
(int)($f['seen'] ?? 0),
|
||||
$f['state'] ?? '?',
|
||||
mb_substr((string)($f['observed'] ?? ''), 0, 44));
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (!vv_ai_repair_enabled()) exit(0); // silent: a disabled feature is not an event
|
||||
|
||||
$lockPath = VV_CACHE_ROOT . '/ai_repair.lock';
|
||||
if (!is_dir(dirname($lockPath))) @mkdir(dirname($lockPath), 0755, true);
|
||||
$lock = @fopen($lockPath, 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
|
||||
if ($lock) fclose($lock);
|
||||
rlog('skipped — a sweep is already running');
|
||||
exit(0);
|
||||
}
|
||||
|
||||
try {
|
||||
$t0 = microtime(true);
|
||||
$sum = vv_ai_repair_sweep($dryRun);
|
||||
$ms = (int)round((microtime(true) - $t0) * 1000);
|
||||
|
||||
if (!($sum['ok'] ?? false)) {
|
||||
rlog('sweep refused — ' . ($sum['error'] ?? 'unknown'));
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// An announcement is worth a line whether or not this pass found anything new — a delivery
|
||||
// that failed on an earlier pass is retried here, and "we tried to tell you" is exactly the
|
||||
// thing someone reads this log to check.
|
||||
$ann = $sum['announced'] ?? [];
|
||||
if (($ann['count'] ?? 0) > 0) {
|
||||
rlog(sprintf('%sannounce: %d finding(s) — %s%s',
|
||||
$dryRun ? 'dry-run: ' : '', $ann['count'],
|
||||
($ann['sent'] ?? false) ? 'sent' : ($dryRun ? 'not sent (dry run)' : 'DELIVERY FAILED'),
|
||||
' — ' . ($ann['subject'] ?? '')));
|
||||
}
|
||||
|
||||
// Nothing found and nothing to say. A line every fifteen minutes reporting no news is how a
|
||||
// log stops being read.
|
||||
if ($sum['findings'] === 0) {
|
||||
if ($sum['runs'] > 0 && $dryRun) rlog(sprintf('dry-run: %d run(s), nothing found (%dms)', $sum['runs'], $ms));
|
||||
exit(0);
|
||||
}
|
||||
|
||||
rlog(sprintf('%s%d run(s): %d finding(s), %d fixed, %d for the operator, %d resolved, %d quiet (%dms)',
|
||||
$dryRun ? 'dry-run: ' : '', $sum['runs'], $sum['findings'], $sum['fixed'],
|
||||
$sum['needs_operator'], $sum['resolved'], $sum['quiet'], $ms));
|
||||
|
||||
foreach ($sum['details'] as $d) rlog(' ' . $d);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
|
||||
exit(0);
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================== AI Repair Sweep ===========================================
|
||||
# ==============================================================================================
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Reads the logs of jobs that finished since the last pass, turns known error shapes into
|
||||
# findings, probes for a correction, and either writes a proven value or leaves the finding
|
||||
# for the operator to answer.
|
||||
# Runs from the watchdog orchestrator. Off unless AI_REPAIR_ENABLED is true.
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
# A one-line shim: exec php on ai_repair_sweep.php in the same directory.
|
||||
# The logic is PHP because everything it needs already is — the conf writer with its backups
|
||||
# and read-back verification, the findings store, and the probe layer are all functions the
|
||||
# WebGUI shares. A bash reimplementation would be a second conf writer, which is precisely the
|
||||
# drift the guarded write path exists to prevent.
|
||||
#
|
||||
# There is no post-run hook in Varaverk; nothing fires when a job finishes. The sweep picks up
|
||||
# completed run records instead, so this is one entry in an orchestrator list rather than a
|
||||
# call added to forty scripts.
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
# A Shim, Not a Program
|
||||
# This file exists only because the scheduler runs shell scripts and the work is PHP.
|
||||
# Anything added here would be logic the WebGUI cannot reach, and the operator answering a
|
||||
# finding in the browser must take exactly the same path as the sweep that filed it.
|
||||
#
|
||||
# Detecting And Repairing Are Separate Trusts
|
||||
# AI_REPAIR_ENABLED alone reads logs, files findings and proposes fixes, writing nothing.
|
||||
# AI_REPAIR_AUTOFIX_ENABLED is what allows a value to be written, and only ever one a probe
|
||||
# has answered on. Both live in master.conf; neither is set by this script.
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
# Never Fatal
|
||||
# Always exits 0 — on a disabled feature, a held lock, or a failed pass. The watchdog
|
||||
# orchestrator runs real work either side of this, and a repair sweep must never be the
|
||||
# reason a cycle reports failure.
|
||||
#
|
||||
# One Sweep At A Time
|
||||
# The PHP takes a non-blocking flock. A pass that overruns its slot cannot have a second
|
||||
# copy start probing and writing conf underneath it.
|
||||
#
|
||||
# Nothing Is Written That Has Not Answered
|
||||
# A value reaches conf only after a probe got a response from it. Toggles are never written
|
||||
# unattended at all — whether something should be switched on is a decision about intent,
|
||||
# and a probe cannot prove intent.
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# AI_ENABLED master switch; nothing here runs without it
|
||||
# AI_REPAIR_ENABLED read logs and file findings
|
||||
# AI_REPAIR_AUTOFIX_ENABLED allow a proven value to be written unattended
|
||||
# AI_PROBE_TIMEOUT seconds a single probe may take
|
||||
# AI_FINDING_RETAIN_DAYS how long closed findings are kept
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ai_repair_sweep.sh
|
||||
# One pass. Files findings, applies proven fixes if autofix is on.
|
||||
#
|
||||
# ai_repair_sweep.sh --dry-run
|
||||
# Probes and reports what it would do. Writes no conf and does not move the marker, so the
|
||||
# same runs are examined again next pass.
|
||||
#
|
||||
# ai_repair_sweep.sh --status
|
||||
# Both switches, when the last pass ran, and every open finding.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/ai_repair_sweep.php" "$@"
|
||||
@@ -0,0 +1,103 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// The mesh entry point for the AI subsystem. Another node forwards an action here over SSH; this
|
||||
// runs it through the same vv_ai_dispatch() a browser request on this node would have used, and
|
||||
// writes the response back as JSON on stdout.
|
||||
//
|
||||
// CLI only. It is never served over HTTP and takes nothing from the environment but stdin.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// stdin {"action":"ask","params":{…},"is_post":true}
|
||||
// stdout {"_vv_rpc":1,"status":200,"body":{…}}
|
||||
//
|
||||
// The wrapper exists so an HTTP status can travel with the body — a 405 raised in the dispatcher
|
||||
// has to arrive at the calling node as a 405, not as a 200 carrying an error string.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// One implementation, two doors.
|
||||
// This shares every handler with api/ai.php through include/ai_actions.php. A mesh request
|
||||
// and a browser request cannot diverge in behaviour because there is only one behaviour.
|
||||
//
|
||||
// The caller's node id is a label, not a claim.
|
||||
// params._vv_node says which node asked, so findings and incidents are stored against the
|
||||
// node they describe. It is not consulted for authorization — possession of the partnership
|
||||
// SSH key already settled that, at onboard.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Refuses to run anywhere but the AI owner.
|
||||
// A mirror that somehow received a forwarded request must not answer it from its own empty
|
||||
// stores. Exits non-zero so the caller reports a transport failure rather than rendering an
|
||||
// empty success.
|
||||
//
|
||||
// Refuses to run over the web.
|
||||
// Guarded on PHP_SAPI. Reachable over HTTP this would be an unauthenticated bypass of every
|
||||
// gate api/ai.php applies, since it takes its whole request from stdin.
|
||||
//
|
||||
// The master switch is honoured here too.
|
||||
// AI_ENABLED false on the owner means the mesh gets the same refusal a local request gets.
|
||||
// The calling node checks its own switch; this one checks the owner's.
|
||||
//
|
||||
// Input is size-capped before it is decoded.
|
||||
// stdin is an untrusted stream from another process. A malformed or endless payload must
|
||||
// fail as a bad request, not as an out-of-memory.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// Not invoked by hand. include/ai_rpc.php opens an SSH session to the owner and runs this
|
||||
// file with the request as JSON on stdin; the response is JSON on stdout. There are no flags
|
||||
// and no arguments — the action, the profile and the payload all arrive in the request body.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/ai_actions.php vv_ai_dispatch() — the shared handlers
|
||||
// include/config.php vv_ai_is_owner()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
if (PHP_SAPI !== 'cli') {
|
||||
http_response_code(404);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
require_once dirname(__DIR__) . '/include/ai_actions.php';
|
||||
|
||||
function vv_rpc_out(int $status, array $body): void {
|
||||
echo json_encode(['_vv_rpc' => 1, 'status' => $status, 'body' => $body],
|
||||
JSON_UNESCAPED_SLASHES), "\n";
|
||||
}
|
||||
|
||||
// Not this node's job. Exit non-zero: the caller must see a transport failure, not an answer
|
||||
// assembled from stores that are empty here by design.
|
||||
if (!vv_ai_is_owner()) {
|
||||
fwrite(STDERR, "ai_rpc: this node is not the AI owner\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
// 1 MiB. A turn's history is the largest legitimate payload and is capped far below this by the
|
||||
// per-message truncation in the dispatcher; anything larger is not a request this serves.
|
||||
$raw = stream_get_contents(STDIN, 1024 * 1024);
|
||||
$req = json_decode((string)$raw, true);
|
||||
if (!is_array($req)) {
|
||||
vv_rpc_out(400, ['ok' => false, 'error' => 'ai_rpc: unreadable request']);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
$action = trim((string)($req['action'] ?? ''));
|
||||
$params = is_array($req['params'] ?? null) ? $req['params'] : [];
|
||||
$isPost = (bool)($req['is_post'] ?? false);
|
||||
|
||||
if ($action === '') {
|
||||
vv_rpc_out(400, ['ok' => false, 'error' => 'ai_rpc: no action']);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// The owner's own switch. The calling node already checked its own; this is the other half, and
|
||||
// it is what makes turning AI off here take it off the whole mesh.
|
||||
if (!vv_ai_enabled()) {
|
||||
vv_rpc_out(200, ['ok' => false, 'error' => 'AI_ENABLED is false on the AI owner — AI features are off']);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
vv_ai_log(sprintf('rpc action=%s from=%s', $action, (string)($params['_vv_node'] ?? '?')));
|
||||
|
||||
$httpStatus = 200;
|
||||
$body = vv_ai_dispatch($action, $params, $isPost, $httpStatus);
|
||||
vv_rpc_out($httpStatus, $body);
|
||||
@@ -1,9 +1,90 @@
|
||||
<?php
|
||||
// API cache writer — runs every minute via Varaverk scheduler.
|
||||
// Builds monitor + arrs payloads and writes them to /tmp/vv_cache/ so page
|
||||
// loads can serve instantly from the file instead of making live HTTP calls.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Background cache writer. Builds the full monitor payload and the arrs payload once a
|
||||
// minute and writes them to the cache, so page loads serve from a file instead of paying
|
||||
// for collection.
|
||||
//
|
||||
// Called by api_cache_writer.sh (bash wrapper required by the scheduler).
|
||||
// OPERATIONAL MODEL
|
||||
// This is what makes the monitor and arrs tabs fast. api/monitor.php and api/arrs.php read
|
||||
// the files this process writes and only fall back to collecting for themselves on a miss.
|
||||
// The expensive work — GraphQL, docker stats, SSH to partners, HTTP to every arr instance —
|
||||
// happens here, on a schedule, off the request path.
|
||||
//
|
||||
// Runs from cron every minute via api_cache_writer.sh, a bash wrapper the scheduler
|
||||
// requires. Not reachable over HTTP, and refuses to run if it ever is.
|
||||
//
|
||||
// The payload assembled here is deliberately identical to api/monitor.php's. The two are
|
||||
// maintained together: a field added there and not here is a field that is only ever served
|
||||
// on a cache miss.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// One API round trip for the whole payload.
|
||||
// vv_api_data() is called once up front and static-cached for the life of the process,
|
||||
// so the API-first collectors below share a single GraphQL query rather than issuing
|
||||
// one each.
|
||||
//
|
||||
// Writes two caches, not one.
|
||||
// monitor and arrs have different consumers and different costs, so they are written
|
||||
// under separate keys and either can be served while the other is stale.
|
||||
//
|
||||
// Every payload carries its own timestamp, so consumers can render age rather than
|
||||
// presenting minute-old numbers as current.
|
||||
//
|
||||
// Reports its own duration on stdout.
|
||||
// The elapsed time goes to the job log, which is the only place a slow collection cycle
|
||||
// becomes visible — this process has no other output and no failure anyone would see.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Refuses to run under a web server.
|
||||
// PHP_SAPI is checked first and a non-CLI invocation is answered with a 404 and no
|
||||
// output. Otherwise a browser hitting this path would trigger the full expensive
|
||||
// collection — including SSH to every partner — outside any cache or rate limit, once
|
||||
// per request.
|
||||
//
|
||||
// Read-only with respect to the system. Every collector observes; none start, stop, or
|
||||
// change anything. The only writes are the two cache files.
|
||||
//
|
||||
// Cache writes are atomic.
|
||||
// vv_cache_write() writes a .tmp and renames, so a page load landing mid-write reads
|
||||
// the previous complete payload rather than a truncated one.
|
||||
//
|
||||
// Every collector degrades to empty rather than fatal.
|
||||
// The library suppresses its filesystem reads and redirects stderr on every shell call,
|
||||
// so absent hardware yields an empty section. On a payload this wide that property is
|
||||
// what keeps one missing subsystem from failing the whole cycle and leaving both caches
|
||||
// to expire.
|
||||
//
|
||||
// A failed cycle is survivable by design.
|
||||
// Nothing here clears the previous cache before building the new one. A run that dies
|
||||
// partway leaves the last good payload in place, and the readers' own age windows —
|
||||
// 300s for monitor, 300s for arrs — are several cycles wide, so a single missed minute
|
||||
// is invisible.
|
||||
//
|
||||
// OUTPUT
|
||||
// VV_CACHE_DIR/monitor.json consumed by api/monitor.php
|
||||
// VV_CACHE_DIR/arrs.json consumed by api/arrs.php
|
||||
// stdout one timing line, captured into the job log
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// api_cache_writer.php
|
||||
// Build both payloads and write them to the cache. Takes no arguments.
|
||||
//
|
||||
// Run every minute by Tools/api_cache_writer.sh, which is the cron entry — this file is the
|
||||
// work, that one is the schedule, the lock and the log.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/monitor.php, include/common.php, include/unraid_api.php,
|
||||
// include/vms.php, include/docker_folders.php, include/arrs.php
|
||||
// Tools/api_cache_writer.sh the bash wrapper cron actually invokes
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
// Collecting this payload means GraphQL, docker stats and SSH to every partner. It must never
|
||||
// be triggerable by an HTTP request.
|
||||
if (PHP_SAPI !== 'cli') {
|
||||
http_response_code(404);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
$_base = dirname(__DIR__);
|
||||
require_once $_base . '/include/monitor.php';
|
||||
@@ -17,8 +98,28 @@ $t = microtime(true);
|
||||
// Call vv_api_data() once — result is static-cached for the rest of this process.
|
||||
vv_api_data();
|
||||
|
||||
// ── AI ────────────────────────────────────────────────────────────────────────
|
||||
// One collection, two consumers. vv_ai_stats() is the expensive part of the AI subsystem —
|
||||
// roughly a second, most of it waiting on Ollama and nvidia-smi — and it is written to its own
|
||||
// cache here so the AI tab's banner, the Scheduler dock and the Monitor row all read the same
|
||||
// numbers from the same moment instead of each paying for their own.
|
||||
//
|
||||
// The monitor block is derived from that same array rather than collected again. Must stay in
|
||||
// step with api/monitor.php's own block: this file is what the Monitor tab normally reads, since
|
||||
// the endpoint only assembles a payload on a cache miss, so a key added there and not here
|
||||
// leaves the card that consumes it loading forever on every ordinary page load and working only
|
||||
// on the one request that happens to miss.
|
||||
$_vv_ai = null;
|
||||
if (vv_ai_ui_on()) {
|
||||
require_once $_base . '/include/ai.php';
|
||||
$_vv_ai_stats = vv_ai_stats();
|
||||
vv_cache_write('ai', $_vv_ai_stats);
|
||||
$_vv_ai = vv_ai_monitor_block($_vv_ai_stats);
|
||||
}
|
||||
|
||||
$monitor = [
|
||||
'system' => vv_system_info(),
|
||||
'varaverk' => vv_varaverk_state(),
|
||||
'fallback' => vv_fallback_state(),
|
||||
'fallback_active' => vv_fallback_active(),
|
||||
'partner' => vv_partner_state(),
|
||||
@@ -44,6 +145,7 @@ $monitor = [
|
||||
'vms' => vv_get_vms(),
|
||||
'docker_folders' => vv_get_docker_folders(),
|
||||
'remote_hosts' => vv_remote_hosts_stats(),
|
||||
'ai' => $_vv_ai,
|
||||
'_api_status' => vv_api_get_status(),
|
||||
'ts' => time(),
|
||||
];
|
||||
@@ -53,5 +155,12 @@ vv_cache_write('monitor', $monitor);
|
||||
$arrs = vv_arrs_all();
|
||||
vv_cache_write('arrs', $arrs);
|
||||
|
||||
// ── Watchdog payload ──────────────────────────────────────────────────────────
|
||||
// Here for the same reason as the other two: vv_wd_all() SSHes to every partner, which cost 8.3s
|
||||
// on this host with the partner unreachable, and the Watchdog tab polls every 30 seconds. Collected off the
|
||||
// request path it is paid once a minute by cron instead of by whoever has the tab open.
|
||||
require_once $_base . '/include/watchdog.php';
|
||||
vv_cache_write('watchdog', vv_wd_all());
|
||||
|
||||
$elapsed = round((microtime(true) - $t) * 1000);
|
||||
echo "Cache written in {$elapsed}ms — monitor + arrs\n";
|
||||
echo "Cache written in {$elapsed}ms — monitor + arrs + watchdog\n";
|
||||
|
||||
@@ -68,3 +68,13 @@
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/api_cache_writer.php"
|
||||
|
||||
# Mesh traffic sample. Rides this job because it needs a steady once-a-minute cadence and adding
|
||||
# a second per-minute cron for one append is more moving parts than the measurement is worth.
|
||||
# Failure is ignored on purpose: a missed sample costs resolution in one window, and this job's
|
||||
# actual purpose is the WebGUI cache.
|
||||
php "$SCRIPT_DIR/mesh_traffic_sample.php" >/dev/null 2>&1 || true
|
||||
|
||||
# Retry any mesh chat that could not be delivered when it was sent — a partner being asleep is
|
||||
# the normal case for the message "my server is going down".
|
||||
php "$SCRIPT_DIR/node_chat_receive.php" --flush >/dev/null 2>&1 || true
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Looks at the whole auth stack the way the Auth tab's two checks look at one thing, and files
|
||||
// what it finds. Two questions, asked of every host NPM serves:
|
||||
//
|
||||
// Is this host actually serving? — the why-check, for anything below threshold
|
||||
// Is this host actually protected? — the access check, for anything behind Authelia
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// Both answers already existed on demand, and both required somebody to open the Auth tab and
|
||||
// press a button on the right row. Nobody does that on a working day. One host here has returned
|
||||
// nothing but 5xx to every request for months, and the hostnames that sit behind an auth_request
|
||||
// block which then waves everyone through are invisible from every page in the plugin, because
|
||||
// the fact is split across NPM, an Authelia config and the directory.
|
||||
//
|
||||
// Findings are the existing answer to "a condition that persists and nobody is looking" — the AI
|
||||
// tab already lists, grades, ages out and notifies on them. This adds the auth stack as a source
|
||||
// rather than inventing a second place for the same idea.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// One pass = read NPM once, then per host: the recorded uptime, and — only where it is warranted
|
||||
// — the live checks. The expensive part is deliberately gated:
|
||||
//
|
||||
// proxy_down filed when 24h uptime is below AUTH_SWEEP_UPTIME_MIN *and* the host has been in
|
||||
// that state longer than AUTH_SWEEP_DOWN_MIN. A restart or a reboot produces a
|
||||
// perfectly ordinary dip, and a finding for every one of those is a list nobody
|
||||
// reads. Live checks run only for hosts that pass this gate.
|
||||
//
|
||||
// access_open filed when a host carries an auth_request block and the Authelia deciding it
|
||||
// reaches its default policy of bypass for a member of no relevant group. No
|
||||
// network calls at all — this is three config files compared.
|
||||
//
|
||||
// Nothing is repaired, restarted or rewritten. This is a reporting pass, and it stays one: the
|
||||
// remedies here are "start a container", "edit a rule", "change a default policy", and every one
|
||||
// of them is a decision rather than a correction.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The gate is time, not count.
|
||||
// A host that has been down for four hours is news whether the probe caught eight samples
|
||||
// or eight hundred. Basing it on samples would file findings at a different threshold on a
|
||||
// node whose probe runs at a different cadence.
|
||||
//
|
||||
// Findings are refreshed, not duplicated.
|
||||
// vv_ai_finding_write() keys on kind + subject + ref, so a host still down tomorrow updates
|
||||
// the record it filed today rather than adding a second one. The store closes what stops
|
||||
// recurring on its own.
|
||||
//
|
||||
// The access check answers for a real user, not a hypothetical one.
|
||||
// "Does the default policy let somebody in" is only meaningful about a person who exists.
|
||||
// The sweep asks about a directory member holding no privileged group, because that is the
|
||||
// account that reveals a rule which protects nothing.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Non-fatal, always. No NPM, no credentials, no domains, AI repair switched off — exits 0.
|
||||
// One pass at a time, flock non-blocking, so a slow pass cannot overlap the next.
|
||||
// Live probing is bounded by the gate above, so a total outage cannot turn one pass into
|
||||
// thirty-five sequential timeouts.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// auth_sweep.php one pass, files findings
|
||||
// auth_sweep.php --dry-run report what it would file, write nothing
|
||||
// auth_sweep.php --report one-screen summary for the Sunday report; silent when clean
|
||||
//
|
||||
// CONFIGURATION
|
||||
// AUTH_SWEEP_ENABLED master switch (default true)
|
||||
// AUTH_SWEEP_UPTIME_MIN 24h percentage below which a host is a candidate (default 96)
|
||||
// AUTH_SWEEP_DOWN_MIN minutes it must have been failing before filing (default 120)
|
||||
// AUTH_SWEEP_ACCESS_CHECK whether to run the protection half at all (default true)
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||
|
||||
$dryRun = in_array('--dry-run', $argv, true);
|
||||
$report = in_array('--report', $argv, true);
|
||||
|
||||
$lock = @fopen(sys_get_temp_dir() . '/vv_auth_sweep.lock', 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
||||
|
||||
try {
|
||||
$v = vv_conf_vars();
|
||||
if (strtolower(trim($v['AUTH_SWEEP_ENABLED'] ?? 'true')) === 'false') {
|
||||
echo "AUTH_SWEEP_ENABLED is false\n"; exit(0);
|
||||
}
|
||||
$minPct = (float) ($v['AUTH_SWEEP_UPTIME_MIN'] ?? 96);
|
||||
$minDown = max(1, (int) ($v['AUTH_SWEEP_DOWN_MIN'] ?? 120)) * 60;
|
||||
$doAccess = strtolower(trim($v['AUTH_SWEEP_ACCESS_CHECK'] ?? 'true')) !== 'false';
|
||||
|
||||
$p = vv_npm_list_proxies();
|
||||
if (!($p['ok'] ?? false)) { echo 'NPM: ' . ($p['error'] ?? 'unreadable') . "\n"; exit(0); }
|
||||
|
||||
$uptime = is_file(vv_auth_db_file('uptime.json'))
|
||||
? (json_decode((string) @file_get_contents(vv_auth_db_file('uptime.json')), true) ?: []) : [];
|
||||
$doms = $uptime['domains'] ?? [];
|
||||
|
||||
$filed = $skipped = 0;
|
||||
$lines = [];
|
||||
|
||||
// ── Half one: hosts that are not serving ──
|
||||
foreach ($p['proxies'] as $h) {
|
||||
// A host switched off is not a fault, it is a decision. Filing one would put a finding on
|
||||
// the list for every host the operator has deliberately parked.
|
||||
if (($h['enabled'] ?? true) === false) continue;
|
||||
|
||||
$names = [];
|
||||
foreach ($h['domain_names'] ?? [] as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
if ($d !== '' && !str_contains($d, '*')) $names[] = $d;
|
||||
}
|
||||
if (!$names) continue;
|
||||
|
||||
// The worst name decides, for the same reason the row does: a host is only as reachable as
|
||||
// its least reachable hostname, and averaging hides one dead name behind three healthy ones.
|
||||
$worstPct = null; $worstDom = ''; $worstRec = null;
|
||||
foreach ($names as $d) {
|
||||
$r = $doms[$d] ?? null;
|
||||
if (!$r) continue;
|
||||
$pct = vv_auth_uptime_window($r['hours'] ?? [], 24);
|
||||
if ($pct === null) continue;
|
||||
if ($worstPct === null || $pct < $worstPct) { $worstPct = $pct; $worstDom = $d; $worstRec = $r; }
|
||||
}
|
||||
if ($worstPct === null || $worstPct >= $minPct) continue;
|
||||
|
||||
// How long it has actually been like this. last_change is when the state last flipped, so
|
||||
// a host that went down two minutes ago is excluded here and caught on a later pass — which
|
||||
// is the whole point of the gate.
|
||||
$since = (int) ($worstRec['last_change'] ?? 0);
|
||||
$for = $since > 0 ? time() - $since : 0;
|
||||
if (($worstRec['state'] ?? '') === 'down' && $for < $minDown) { $skipped++; continue; }
|
||||
|
||||
$why = vv_npm_why((int) ($h['id'] ?? 0));
|
||||
if (!($why['ok'] ?? false)) continue;
|
||||
|
||||
// The findings the check already writes, which is the whole reason this does not have its
|
||||
// own opinion about what is wrong. Only the decisive ones are carried into the record.
|
||||
$said = [];
|
||||
foreach ($why['findings'] as $f) if (in_array($f['level'], ['bad', 'warn'], true)) $said[] = $f['text'];
|
||||
if (!$said) { $skipped++; continue; }
|
||||
|
||||
$evidence = sprintf("%s is at %.2f%% over 24h%s.\n\n%s",
|
||||
$worstDom, $worstPct,
|
||||
$for > 0 ? ' and has been ' . ($worstRec['state'] ?? 'failing') . ' for ' . round($for / 3600, 1) . ' hours' : '',
|
||||
implode("\n", $said));
|
||||
|
||||
$lines[] = sprintf(' %-34s %6.2f%% %s', $worstDom, $worstPct, $said[0]);
|
||||
if ($dryRun) { $filed++; continue; }
|
||||
|
||||
$w = vv_ai_finding_write([
|
||||
'kind' => 'proxy_down',
|
||||
'subject' => implode(', ', $names),
|
||||
'ref' => 'npm:proxy:' . ($h['id'] ?? 0),
|
||||
'evidence' => $evidence,
|
||||
'observed' => sprintf('%.2f%% over 24h', $worstPct),
|
||||
// Proven, because these are measurements rather than an inference: a TCP connect either
|
||||
// completed or it did not, and the access log either counted 5xx or it did not.
|
||||
'proven' => true,
|
||||
]);
|
||||
$w['ok'] ? $filed++ : $skipped++;
|
||||
}
|
||||
|
||||
// ── Half two: hosts that are guarded but not protected ──
|
||||
//
|
||||
// Grouped by Authelia instance, not filed per hostname. The first version of this produced
|
||||
// twenty-two findings that were all the same sentence, because they all had the same cause: a
|
||||
// default policy of bypass means every host whose rule does not name your group lets you
|
||||
// through, so the number of findings was really the number of hostnames. One finding per
|
||||
// instance, naming the hosts it affects, is the fact — and it has one fix rather than
|
||||
// twenty-two.
|
||||
$openLines = [];
|
||||
if ($doAccess) {
|
||||
// Somebody who exists and holds none of the groups the rules name. A rule that still lets
|
||||
// this account through is a rule protecting nothing, and asking about an invented username
|
||||
// would prove nothing about the directory.
|
||||
$probe = vv_auth_sweep_ordinary_user();
|
||||
$byInstance = [];
|
||||
|
||||
foreach ($p['proxies'] as $h) {
|
||||
if (($h['enabled'] ?? true) === false) continue;
|
||||
if (!str_contains((string) ($h['advanced_config'] ?? ''), 'auth_request')) continue;
|
||||
|
||||
foreach ($h['domain_names'] ?? [] as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
if ($d === '' || str_contains($d, '*')) continue;
|
||||
|
||||
$a = vv_auth_access_check($d, $probe);
|
||||
if (!($a['ok'] ?? false)) continue;
|
||||
if (($a['policy'] ?? '') !== 'bypass') continue;
|
||||
|
||||
$inst = (string) ($a['authelia']['container'] ?? '?');
|
||||
$byInstance[$inst]['default'] = (string) ($a['default_policy'] ?? '?');
|
||||
$byInstance[$inst]['config'] = (string) ($a['authelia']['config'] ?? '');
|
||||
// Which of the two shapes this is, per host: a hostname no rule mentions, or one a
|
||||
// rule covers and then steps over. They have different fixes — write a rule, or
|
||||
// widen an existing one — so the record keeps them apart.
|
||||
$stepped = false;
|
||||
foreach ($a['trace'] ?? [] as $t) if (($t['skip'] ?? '') === 'subject') $stepped = true;
|
||||
$byInstance[$inst][$stepped ? 'stepped' : 'unlisted'][] = $d;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($byInstance as $inst => $g) {
|
||||
$unlisted = $g['unlisted'] ?? [];
|
||||
$stepped = $g['stepped'] ?? [];
|
||||
$all = array_merge($unlisted, $stepped);
|
||||
if (!$all) continue;
|
||||
|
||||
$ev = [];
|
||||
$ev[] = $inst . ' has default_policy: ' . ($g['default'] ?? '?') . ', so any request its rules do '
|
||||
. 'not decide is allowed through. ' . count($all) . ' hostname'
|
||||
. (count($all) === 1 ? '' : 's') . ' behind an auth_request block pointing at it reach the '
|
||||
. 'application without being asked to authenticate.';
|
||||
if ($unlisted) $ev[] = "\nNo rule mentions these at all:\n " . implode("\n ", $unlisted);
|
||||
if ($stepped) $ev[] = "\nA rule covers these but does not apply to an ordinary account"
|
||||
. ($probe ? ' (tested as ' . $probe . ')' : '') . ":\n " . implode("\n ", $stepped);
|
||||
$ev[] = "\nRules are in " . ($g['config'] ?: 'a config that was not found') . '.';
|
||||
// Said plainly because "bypass" reads as harmless and it is the single most
|
||||
// consequential line in that file.
|
||||
$ev[] = "\nThe fix is a default_policy of deny with an explicit rule for anything that is "
|
||||
. "meant to be public — not a rule per hostname above.";
|
||||
|
||||
$openLines[] = sprintf(' %-22s default bypass — %d hostname%s unprotected',
|
||||
$inst, count($all), count($all) === 1 ? '' : 's');
|
||||
if ($dryRun) { $filed++; continue; }
|
||||
|
||||
$w = vv_ai_finding_write([
|
||||
'kind' => 'access_open',
|
||||
'subject' => $inst . ' — ' . count($all) . ' hostnames not protected',
|
||||
'ref' => 'authelia:' . $inst . ':default_policy',
|
||||
'evidence' => implode("\n", $ev),
|
||||
'observed' => 'default_policy: ' . ($g['default'] ?? '?'),
|
||||
'proven' => true,
|
||||
]);
|
||||
$w['ok'] ? $filed++ : $skipped++;
|
||||
}
|
||||
}
|
||||
|
||||
if ($report) {
|
||||
// Silent on a clean week. The orchestrator's job is to say nothing when there is nothing
|
||||
// to say, and a section that always prints is a section that stops being read.
|
||||
if (!$lines && !$openLines) exit(0);
|
||||
echo "Auth stack review\n";
|
||||
if ($lines) { echo "\nProxy hosts not serving:\n"; foreach ($lines as $l) echo "$l\n"; }
|
||||
if ($openLines) { echo "\nBehind Authelia but not protected:\n"; foreach ($openLines as $l) echo "$l\n"; }
|
||||
exit(1);
|
||||
}
|
||||
|
||||
printf("%s%d finding%s, %d skipped\n", $dryRun ? 'dry run — ' : '', $filed, $filed === 1 ? '' : 's', $skipped);
|
||||
foreach (array_merge($lines, $openLines) as $l) echo "$l\n";
|
||||
exit(0);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
|
||||
// A directory member holding none of the groups any rule names. Returns '' when every user is
|
||||
// privileged or the directory cannot be read, and the caller then asks about no user at all —
|
||||
// which still answers the "no rule mentions this host" case and simply cannot answer the
|
||||
// "the rule stepped over this person" one.
|
||||
function vv_auth_sweep_ordinary_user(): string {
|
||||
$named = [];
|
||||
// Groups named by the rules of every Authelia instance in play, not just the configured one —
|
||||
// a .us hostname is decided by a config this conf file does not point at.
|
||||
foreach (vv_auth_sweep_configs() as $cfg) {
|
||||
$r = vv_authelia_read_rules($cfg);
|
||||
foreach (($r['ok'] ?? false) ? $r['rules'] : [] as $rule) {
|
||||
$s = $rule['subject'] ?? null;
|
||||
foreach (is_array($s) ? $s : [$s] as $alt)
|
||||
foreach (is_array($alt) ? $alt : [$alt] as $one)
|
||||
if (is_string($one) && str_starts_with($one, 'group:')) $named[strtolower(substr($one, 6))] = true;
|
||||
}
|
||||
}
|
||||
|
||||
$u = vv_lldap_list_users();
|
||||
foreach (($u['ok'] ?? false) ? $u['users'] : [] as $user) {
|
||||
$mine = array_map('strtolower', array_filter(array_column($user['groups'] ?? [], 'displayName')));
|
||||
if (array_intersect($mine, array_keys($named))) continue;
|
||||
return (string) ($user['id'] ?? '');
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
// Every Authelia configuration this installation actually uses, discovered through the proxy hosts
|
||||
// rather than listed anywhere. Two instances run here and conf names one.
|
||||
function vv_auth_sweep_configs(): array {
|
||||
$out = [];
|
||||
$p = vv_npm_list_proxies();
|
||||
foreach (($p['ok'] ?? false) ? $p['proxies'] : [] as $h) {
|
||||
$i = vv_authelia_instance_for($h);
|
||||
if (($i['config'] ?? '') !== '' && is_file($i['config'])) $out[$i['config']] = true;
|
||||
}
|
||||
return array_keys($out);
|
||||
}
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ===================================== Auth Sweep =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Asks the two questions the Auth tab can answer about one host, about every host, and files
|
||||
# what it finds as findings.
|
||||
#
|
||||
# Is this host serving? below the uptime threshold, and failing for longer than a restart
|
||||
# Is this host protected? behind an auth_request block that no rule then applies to
|
||||
#
|
||||
# Both answers existed already and both needed somebody to open the tab and press a button on
|
||||
# the right row. One host here has returned nothing but 5xx for months.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in auth_sweep.php.
|
||||
#
|
||||
# Reports only — nothing is started, restarted or rewritten. The remedies are "start a
|
||||
# container", "edit a rule", "change a default policy", and each of those is a decision.
|
||||
#
|
||||
# The live half is gated on time rather than on sample count: a host must have been failing for
|
||||
# longer than AUTH_SWEEP_DOWN_MIN before anything is filed, so a reboot does not produce a
|
||||
# finding for every hostname on the machine.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# It answers the questions nobody was going to ask.
|
||||
# Both checks were already possible from the Auth tab, and both required somebody to open it
|
||||
# and press a button on the right row. That is why one host here returned nothing but 5xx for
|
||||
# months. A check that depends on being run is a check that is not running.
|
||||
#
|
||||
# Files findings; never applies remedies.
|
||||
# The fixes are "start a container", "edit a rule", "change a default policy". Each of those
|
||||
# is a judgement about intent, and a sweep that guessed at intent would be repairing a
|
||||
# deliberate configuration as often as a broken one.
|
||||
#
|
||||
# Down is measured in time, not in samples.
|
||||
# A host must have been failing for longer than AUTH_SWEEP_DOWN_MIN before anything is filed.
|
||||
# Counting failed samples instead would file a finding for every hostname on the machine every
|
||||
# time it reboots, and a report that cries wolf after every restart stops being read.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only against the auth stack. Nothing here starts a container, edits a proxy host, changes
|
||||
# a rule or alters a default policy — the two questions are answered by observation alone.
|
||||
#
|
||||
# --dry-run reports what it would file and writes no finding. --report only reads what is
|
||||
# already filed, and is silent when clean, so it cannot manufacture noise for the Sunday report.
|
||||
#
|
||||
# Findings are filed, not notified. A sweep that raised an alert per host would be its own
|
||||
# incident every time the array restarted.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# auth_sweep.sh one pass, files findings
|
||||
# auth_sweep.sh --dry-run report what it would file, write nothing
|
||||
# auth_sweep.sh --report one-screen summary for the Sunday report; silent when clean
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# AUTH_SWEEP_ENABLED master switch
|
||||
# AUTH_SWEEP_UPTIME_MIN 24h percentage below which a host is a candidate
|
||||
# AUTH_SWEEP_DOWN_MIN minutes it must have been failing before a finding is filed
|
||||
# AUTH_SWEEP_ACCESS_CHECK whether to run the protection half at all
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/auth_sweep.php" "$@"
|
||||
@@ -0,0 +1,292 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Keeps a running record of every certificate NPM holds: when it was first seen, how many times
|
||||
// it has renewed, how many times it has been found expired, and how long it has been tracked.
|
||||
// The Certs tab reads what this writes; nothing here draws anything.
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// NPM knows what a certificate is today and nothing about what it was. cert_monitor.sh checks
|
||||
// the two domains in CERT_MONITOR_DOMAINS over TLS and writes a snapshot that the next run
|
||||
// overwrites. So "has this domain been renewing cleanly for the last year" had no answer
|
||||
// anywhere, and the ten certificates that had been failing renewal for months — 1001 failed
|
||||
// certbot runs, zero successes — were visible only by reading certbot's logs by hand.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// One pass per invocation, over the certificate list NPM returns:
|
||||
// 1. A domain not in the store is added, with first_seen taken from NPM's created_on rather
|
||||
// than from now — that date is real and this file should not pretend tracking began the
|
||||
// day it was installed.
|
||||
// 2. An expiry that has moved later than the stored one is a renewal.
|
||||
// 3. An expiry in the past is a failure, and a strike.
|
||||
// 4. CERT_HISTORY_STRIKES strikes retires the domain from the active list. It stays in the
|
||||
// store — the history is the point — but stops being counted as something that works.
|
||||
// 5. A tracked domain that is no longer in NPM is marked removed, not struck. Deleting a
|
||||
// certificate is a decision; failing to renew one is not.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Counts start at zero and are only ever real.
|
||||
// There is no renewal history anywhere on this host to seed from — every Let's Encrypt
|
||||
// archive directory holds exactly one generation, so no renewal has ever completed through
|
||||
// this NPM. Back-filling a plausible number would make the card a guess wearing a
|
||||
// statistic. first_seen is seeded because it is a fact NPM already holds.
|
||||
//
|
||||
// Keyed by domain, not by NPM id.
|
||||
// A certificate deleted and re-issued gets a new id and is the same domain. Keying on the
|
||||
// id would restart the history of anything ever recreated, which is exactly the moment the
|
||||
// history is worth having.
|
||||
//
|
||||
// A renewal is an expiry that moved forward.
|
||||
// Derived from the list NPM already returns rather than from a TLS handshake per domain:
|
||||
// forty openssl connections to answer a question the API has already answered is a lot of
|
||||
// runtime for the same fact. cert_monitor.sh still does the handshake for the domains that
|
||||
// need the outside world's view.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Non-fatal, always. A missing NPM, bad credentials or an unreadable store exits 0 with a
|
||||
// message. This runs inside the Sunday report and must never be the reason it fails.
|
||||
//
|
||||
// One pass at a time — flock, non-blocking.
|
||||
//
|
||||
// The store is written atomically, tmp + rename, and a write that cannot be verified leaves
|
||||
// the previous file in place. This is append-mostly history; a truncated write loses all of it.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// cert_history.php one pass, updates the store
|
||||
// cert_history.php --dry-run reports what it would change, writes nothing
|
||||
// cert_history.php --status prints the store as a table
|
||||
//
|
||||
// CONFIGURATION
|
||||
// CERT_HISTORY_STRIKES consecutive failed passes before a domain is retired (default 5)
|
||||
// DB_DIR cert_history.json lives here
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
$dryRun = in_array('--dry-run', $argv, true);
|
||||
$status = in_array('--status', $argv, true);
|
||||
|
||||
const VV_CERT_HISTORY_FILE = 'cert_history.json';
|
||||
|
||||
function vv_cert_history_path(): string {
|
||||
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/' . VV_CERT_HISTORY_FILE;
|
||||
}
|
||||
|
||||
function vv_cert_history_read(): array {
|
||||
$p = vv_cert_history_path();
|
||||
if (!is_file($p)) return ['domains' => [], 'created' => time()];
|
||||
$j = json_decode((string) @file_get_contents($p), true);
|
||||
// A corrupt store is not overwritten from here — it is reported and left alone, because the
|
||||
// alternative is a pass that silently restarts every counter from zero.
|
||||
if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return [];
|
||||
return $j;
|
||||
}
|
||||
|
||||
function vv_cert_history_write(array $data): bool {
|
||||
$p = vv_cert_history_path();
|
||||
$dir = dirname($p);
|
||||
if (!is_dir($dir) && !@mkdir($dir, 0755, true)) return false;
|
||||
$data['updated'] = time();
|
||||
$json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
|
||||
if ($json === false) return false;
|
||||
$tmp = $p . '.vv.tmp';
|
||||
if (@file_put_contents($tmp, $json) === false) return false;
|
||||
// Verified before it replaces the real file. This is the only copy of the history.
|
||||
if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
|
||||
if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
function vv_cert_strike_limit(): int {
|
||||
$n = (int) (vv_conf_vars()['CERT_HISTORY_STRIKES'] ?? 5);
|
||||
return $n > 0 ? $n : 5;
|
||||
}
|
||||
|
||||
// ── Status ────────────────────────────────────────────────────────────────────
|
||||
if ($status) {
|
||||
$h = vv_cert_history_read();
|
||||
if (!$h) { echo "cert_history.json is unreadable or malformed\n"; exit(0); }
|
||||
$d = $h['domains'] ?? [];
|
||||
printf("%-32s %7s %7s %7s %7s %-10s %s\n",
|
||||
'domain', 'checks', 'renews', 'fails', 'strikes', 'expires', 'tracked');
|
||||
foreach ($d as $dom => $r) {
|
||||
printf("%-32s %7d %7d %7d %7d %-10s %s%s\n", substr($dom, 0, 32),
|
||||
$r['checks'] ?? 0, $r['renewals'] ?? 0, $r['failures'] ?? 0, $r['strikes'] ?? 0,
|
||||
$r['last_expiry'] ?? '-', vv_cert_span($r['first_seen'] ?? time()),
|
||||
!empty($r['retired_at']) ? ' RETIRED' : (!empty($r['removed_at']) ? ' removed' : ''));
|
||||
}
|
||||
printf("\n%d tracked, strike limit %d\n", count($d), vv_cert_strike_limit());
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Years, months and days rather than a day count. "3 years 6 months and 22 days" is the shape the
|
||||
// question is asked in; 1298 days is the same fact in a unit nobody thinks in.
|
||||
function vv_cert_span(int $from, ?int $to = null): string {
|
||||
$a = (new DateTime())->setTimestamp($from);
|
||||
$b = (new DateTime())->setTimestamp($to ?? time());
|
||||
if ($b < $a) return '0d';
|
||||
$d = $a->diff($b);
|
||||
$out = [];
|
||||
if ($d->y) $out[] = $d->y . 'y';
|
||||
if ($d->m) $out[] = $d->m . 'mo';
|
||||
if ($d->d || !$out) $out[] = $d->d . 'd';
|
||||
return implode(' ', $out);
|
||||
}
|
||||
|
||||
// What one observation of one domain does to its record. Pure — takes the record and the facts,
|
||||
// returns the new record and what happened — so the strike ladder can be tested without waiting
|
||||
// for a certificate to expire. That mattered: nothing on this host is expired right now, so the
|
||||
// failure branch would otherwise ship having never run.
|
||||
//
|
||||
// $exp the certificate's expiry, as a timestamp
|
||||
// $now the moment of this pass
|
||||
// $limit strikes before retirement
|
||||
function vv_cert_apply(array $r, int $exp, int $now, int $limit): array {
|
||||
$out = ['renewed' => false, 'failed' => false, 'retired' => false, 'from' => '', 'to' => ''];
|
||||
|
||||
$r['checks'] = ($r['checks'] ?? 0) + 1;
|
||||
$r['last_seen'] = $now;
|
||||
// Cleared on sight: a domain that is back in NPM is not removed any more, whatever it was
|
||||
// last pass.
|
||||
$r['removed_at'] = null;
|
||||
|
||||
// Both sides reduced to a date. last_expiry is stored as Y-m-d and NPM's expires_on carries a
|
||||
// time, so comparing raw timestamps made every re-read of the same certificate look like a
|
||||
// renewal to a few hours later.
|
||||
$expDay = strtotime(date('Y-m-d', $exp));
|
||||
$prev = !empty($r['last_expiry']) ? strtotime($r['last_expiry']) : null;
|
||||
|
||||
if ($prev !== null && $expDay > $prev) {
|
||||
$r['renewals'] = ($r['renewals'] ?? 0) + 1;
|
||||
$r['last_renewal'] = $now;
|
||||
// A renewal clears the strikes and un-retires. The point of a strike count is "how long
|
||||
// has this been broken", and it is no longer broken.
|
||||
$r['strikes'] = 0;
|
||||
$r['retired_at'] = null;
|
||||
$out['renewed'] = true;
|
||||
$out['from'] = date('Y-m-d', $prev);
|
||||
$out['to'] = date('Y-m-d', $expDay);
|
||||
}
|
||||
|
||||
if ($exp < $now) {
|
||||
$r['failures'] = ($r['failures'] ?? 0) + 1;
|
||||
$r['strikes'] = ($r['strikes'] ?? 0) + 1;
|
||||
$out['failed'] = true;
|
||||
if ($r['strikes'] >= $limit && empty($r['retired_at'])) {
|
||||
$r['retired_at'] = $now;
|
||||
$r['retired_reason'] = "expired for {$r['strikes']} consecutive passes";
|
||||
$out['retired'] = true;
|
||||
}
|
||||
}
|
||||
|
||||
$r['last_expiry'] = date('Y-m-d', $expDay);
|
||||
$out['record'] = $r;
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── One pass ──────────────────────────────────────────────────────────────────
|
||||
$lockPath = sys_get_temp_dir() . '/vv_cert_history.lock';
|
||||
$lock = @fopen($lockPath, 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) {
|
||||
echo "another pass is running\n";
|
||||
exit(0);
|
||||
}
|
||||
|
||||
try {
|
||||
if ($miss = vv_auth_creds_missing('npm')) { echo "$miss\n"; exit(0); }
|
||||
|
||||
$certs = vv_npm_list_certs();
|
||||
if (!$certs) { echo "NPM returned no certificates — nothing to record\n"; exit(0); }
|
||||
|
||||
$hist = vv_cert_history_read();
|
||||
if (!$hist) { echo "cert_history.json is malformed — refusing to overwrite it\n"; exit(1); }
|
||||
$store = $hist['domains'] ?? [];
|
||||
$limit = vv_cert_strike_limit();
|
||||
$now = time();
|
||||
$seen = [];
|
||||
$added = $renewed = $failed = $retired = $removed = 0;
|
||||
$notes = [];
|
||||
|
||||
// Resolved to one certificate per domain before anything is counted. A domain can appear on
|
||||
// more than one certificate — three do here, left behind by re-issuing rather than replacing —
|
||||
// and walking the list directly counted each of them as a separate check of the same domain,
|
||||
// then read the second one's expiry as a renewal of the first.
|
||||
//
|
||||
// The winner is the latest expiry, because that is the one actually worth serving; the earliest
|
||||
// created is kept as first_seen for the same reason NPM's created_on is used at all.
|
||||
$byDomain = [];
|
||||
foreach ($certs as $c) {
|
||||
$exp = !empty($c['expires_on']) ? strtotime((string) $c['expires_on']) : false;
|
||||
if ($exp === false) continue;
|
||||
$created = !empty($c['created_on']) ? strtotime((string) $c['created_on']) : $now;
|
||||
foreach (($c['domain_names'] ?? []) as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
if ($d === '') continue;
|
||||
if (!isset($byDomain[$d]) || $exp > $byDomain[$d]['exp'])
|
||||
$byDomain[$d] = ['exp' => $exp, 'cert' => $c, 'created' => $created];
|
||||
else
|
||||
$byDomain[$d]['created'] = min($byDomain[$d]['created'], $created ?: $now);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
foreach ($byDomain as $domain => $info) {
|
||||
$exp = $info['exp'];
|
||||
$c = $info['cert'];
|
||||
$seen[$domain] = true;
|
||||
|
||||
if (!isset($store[$domain])) {
|
||||
$created = $info['created'];
|
||||
$store[$domain] = [
|
||||
'first_seen' => $created !== false ? $created : $now,
|
||||
'seeded_from' => 'npm_created_on',
|
||||
'checks' => 0, 'renewals' => 0, 'failures' => 0, 'strikes' => 0,
|
||||
'last_expiry' => null, 'last_renewal' => null,
|
||||
'retired_at' => null, 'removed_at' => null,
|
||||
];
|
||||
$added++;
|
||||
$notes[] = "added $domain (first seen " . date('Y-m-d', $store[$domain]['first_seen']) . ')';
|
||||
}
|
||||
|
||||
$r = &$store[$domain];
|
||||
$r['npm_id'] = $c['id'] ?? null;
|
||||
$r['provider'] = $c['provider'] ?? null;
|
||||
|
||||
$res = vv_cert_apply($r, $exp, $now, $limit);
|
||||
$r = $res['record'];
|
||||
if ($res['renewed']) { $renewed++; $notes[] = "renewed $domain ($res[from] → $res[to])"; }
|
||||
if ($res['failed']) { $failed++; }
|
||||
if ($res['retired']) { $retired++; $notes[] = "RETIRED $domain after {$r['strikes']} strikes"; }
|
||||
unset($r);
|
||||
}
|
||||
}
|
||||
|
||||
// Tracked but no longer in NPM. Marked, never struck and never deleted from the store — the
|
||||
// history of a domain that used to exist is the reason this file is kept.
|
||||
foreach ($store as $domain => &$r) {
|
||||
if (isset($seen[$domain])) continue;
|
||||
if (empty($r['removed_at'])) {
|
||||
$r['removed_at'] = $now;
|
||||
$removed++;
|
||||
$notes[] = "no longer in NPM: $domain";
|
||||
}
|
||||
}
|
||||
unset($r);
|
||||
|
||||
ksort($store);
|
||||
$hist['domains'] = $store;
|
||||
$hist['last_pass'] = $now;
|
||||
|
||||
printf("%d certificates, %d domains tracked — added %d, renewed %d, failed %d, retired %d, removed %d\n",
|
||||
count($certs), count($store), $added, $renewed, $failed, $retired, $removed);
|
||||
foreach ($notes as $n) echo " $n\n";
|
||||
|
||||
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
||||
if (!vv_cert_history_write($hist)) { echo "could not write " . vv_cert_history_path() . "\n"; exit(1); }
|
||||
echo 'wrote ' . vv_cert_history_path() . "\n";
|
||||
exit(0);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
Executable
+79
@@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================ Certificate History =========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Records what happens to every certificate NPM holds — first seen, renewals, failures, and how
|
||||
# long each domain has been tracked — into DB_DIR/cert_history.json. The Certs tab reads it.
|
||||
#
|
||||
# NPM knows what a certificate is today and nothing about what it was, and cert_monitor.sh writes
|
||||
# a snapshot the next run overwrites. Ten certificates on this host had been failing renewal for
|
||||
# months without anything on any page being able to say so.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in cert_history.php, next to the NPM client it needs — the API token
|
||||
# handling lives in include/auth.php and reimplementing it in bash to avoid a php call would be a
|
||||
# second copy of the thing most worth having only one of. Same split as api_cache_writer and
|
||||
# ai_repair_sweep.
|
||||
#
|
||||
# Counts start from zero on first run and are only ever observed. first_seen is seeded from NPM's
|
||||
# own created_on, which is a real date; nothing else is back-filled.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# History is accumulated, never reconstructed.
|
||||
# NPM holds only the present, so every past state this file knows about is one it observed at
|
||||
# the time. Counts start at zero on first run and nothing is back-filled — first_seen is the
|
||||
# single exception, seeded from NPM's own created_on because that is a real recorded date
|
||||
# rather than an inference.
|
||||
#
|
||||
# A domain is retired on strikes, not on a single bad pass.
|
||||
# A pass fails for a domain when it is absent from NPM's list or its expiry is already in the
|
||||
# past — and NPM's list can come back short for reasons that have nothing to do with the
|
||||
# certificate, such as an API hiccup or a restart mid-pass. CERT_HISTORY_STRIKES consecutive
|
||||
# failures are required before a domain is retired, and it stays in the store afterwards, so
|
||||
# one bad read can neither erase months of history nor hide a genuine expiry.
|
||||
#
|
||||
# The wrapper holds no logic.
|
||||
# The work sits next to the NPM client it needs, because token handling lives in
|
||||
# include/auth.php and a bash reimplementation would be a second copy of the thing most worth
|
||||
# having only one of. Flags are forwarded verbatim.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# The store is replaced atomically: temp file, verified, then renamed over the original. A pass
|
||||
# that fails partway leaves the previous history intact rather than a truncated file — this is
|
||||
# the only record of what these certificates did, and there is no second copy to restore from.
|
||||
#
|
||||
# --dry-run reports every change it would make and writes nothing. --status only reads.
|
||||
#
|
||||
# Read-only against NPM. Certificates are observed; nothing here renews, deletes or edits one.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# cert_history.sh one pass, updates the store
|
||||
# cert_history.sh --dry-run reports what it would change, writes nothing
|
||||
# cert_history.sh --status prints the store as a table
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# CERT_HISTORY_STRIKES consecutive failed passes before a domain is retired (default 5)
|
||||
# DB_DIR cert_history.json is written here
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/cert_history.php" "$@"
|
||||
@@ -0,0 +1,93 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Reads certbot's own logs and names why renewals failed, in the handful of categories they
|
||||
// actually fall into — rather than leaving 639 MB of Python tracebacks as the only record.
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// Tools/cert_history.sh counts failures. It infers them from an expiry in the past, so it knows
|
||||
// that a domain stopped renewing and nothing about why. The why is in certbot's log, which on
|
||||
// this installation is 1001 rotated files, and the answer to "why did ten certificates stop
|
||||
// renewing" was previously a person reading them by hand.
|
||||
//
|
||||
// The categories matter more than the count, because they are not independent. Missing DNS
|
||||
// produces a failure; the failure is retried; the retries exhaust Let's Encrypt's rate limit;
|
||||
// and the rate limit then fails every *other* domain too. A count says "2079 rate limit errors"
|
||||
// and points at the symptom. The chain says "three hostnames have no DNS records, and that is
|
||||
// what burned the rate limit for everything else".
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Reads the newest N log files and classifies each one. One file is one certbot run, and a run
|
||||
// is what gets counted — a single failure writes its reason into the ACME response, the Python
|
||||
// traceback and certbot's own summary, so counting lines reports one failure as three and makes
|
||||
// the noisier categories look larger than the quiet ones. Bounded three ways, because this can
|
||||
// be called from a page request:
|
||||
//
|
||||
// files CERT_TRIAGE_FILES, newest first
|
||||
// bytes CERT_TRIAGE_MAX_BYTES per file, read from the end
|
||||
// order by the numeric rotation suffix, never by mtime
|
||||
//
|
||||
// The suffix is load-bearing. Every one of these files carries the same mtime here — they are
|
||||
// synced as a set, so the filesystem timestamps say they were all written at once. Sorting by
|
||||
// mtime would pick an arbitrary thousand-file-old sample and report it as current.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Classify, never guess. A run that errored and matched no known pattern is counted as
|
||||
// unclassified and said so, rather than being folded into the nearest category — an "unknown"
|
||||
// that is honest is worth more than a tidy chart that is wrong.
|
||||
//
|
||||
// The root causes and the consequences are reported separately. Rate limiting is almost always
|
||||
// downstream of something else here, and listing it alongside its own cause invites fixing the
|
||||
// symptom.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Opens certbot's logs and nothing else — no certificate is requested, renewed or
|
||||
// deleted here, and no log is rotated or truncated.
|
||||
//
|
||||
// Bounded by file count and by bytes per file, so a directory that has grown to 639 MB across
|
||||
// 1001 files cannot turn a page load into an unbounded read. Only the tail of each log is
|
||||
// examined, because a run explains its failure at the end rather than the beginning.
|
||||
//
|
||||
// An unrecognised failure is reported as unclassified, never folded into the nearest category.
|
||||
// A tidy chart that is wrong sends the operator to fix a domain that was never broken.
|
||||
//
|
||||
// The log directory is discovered from the NPM container rather than assumed, so a container
|
||||
// path change surfaces as "no logs found" instead of an empty triage that reads as "no
|
||||
// failures".
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// cert_triage.php summary — categories, affected domains, and the causal reading
|
||||
// cert_triage.php --json the same as JSON, for the Certs tab
|
||||
// cert_triage.php --files=N override how many rotated logs to read
|
||||
//
|
||||
// CONFIGURATION
|
||||
// CERT_TRIAGE_FILES rotated logs to read, newest first (default 40)
|
||||
// CERT_TRIAGE_MAX_BYTES bytes read from the end of each (default 262144)
|
||||
// CERT_TRIAGE_LOG_DIR override the log directory; normally found from the NPM container
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
$json = in_array('--json', $argv, true);
|
||||
$filesOverride = 0;
|
||||
foreach ($argv as $a) if (preg_match('/^--files=(\d+)$/', $a, $m)) $filesOverride = (int) $m[1];
|
||||
|
||||
$r = vv_cert_triage($filesOverride);
|
||||
|
||||
if ($json) { echo json_encode($r), "\n"; exit(0); }
|
||||
|
||||
if (!($r['ok'] ?? false)) { echo ($r['error'] ?? 'failed'), "\n"; exit(0); }
|
||||
|
||||
printf("%d certbot runs read, %d failed, %d of those matched nothing known\n\n",
|
||||
$r['files_read'], $r['total'], $r['unclassified']);
|
||||
|
||||
if (!$r['total']) { echo "No renewal failures in the logs read.\n"; exit(0); }
|
||||
|
||||
foreach ($r['categories'] as $c) {
|
||||
printf(" %-22s %5d %s\n", $c['id'], $c['count'], $c['what']);
|
||||
foreach (array_slice($c['domains'], 0, 6) as $d) printf(" %s\n", $d);
|
||||
if (count($c['domains']) > 6) printf(" … and %d more\n", count($c['domains']) - 6);
|
||||
}
|
||||
|
||||
if ($r['reading']) { echo "\n"; foreach ($r['reading'] as $l) echo " $l\n"; }
|
||||
exit(0);
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ==================================== Cert Triage =============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Reads certbot's own logs and names why renewals failed, in the handful of categories they
|
||||
# actually fall into.
|
||||
#
|
||||
# cert_history.sh counts failures — it notices an expiry in the past. It cannot say why. The why
|
||||
# is in certbot's log, which here is 1001 rotated files and 639 MB, and the last time anyone
|
||||
# answered "why did ten certificates stop renewing" they read them by hand.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in cert_triage.php.
|
||||
#
|
||||
# Counts runs, not lines. One log file is one certbot invocation, and one failure writes its
|
||||
# reason three times — in the ACME response, the traceback, and certbot's own summary. Counting
|
||||
# lines reports a single failure as three and inflates whichever category is most verbose.
|
||||
#
|
||||
# Reads the newest logs by rotation suffix, never by mtime. Every file here carries the same
|
||||
# mtime because they are synced as a set, so mtime order is meaningless.
|
||||
#
|
||||
# The categories are separated into causes and consequences. Rate limiting is nearly always
|
||||
# downstream — retries against a hostname with no DNS record exhaust the allowance, which then
|
||||
# fails renewals for domains that have nothing wrong with them.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# One invocation is one data point.
|
||||
# A single failure writes its reason three times — the ACME response, the traceback, and
|
||||
# certbot's summary. Counting lines would report it as three and inflate whichever category
|
||||
# happens to be the most verbose, which is the opposite of what triage is for.
|
||||
#
|
||||
# Causes are separated from consequences.
|
||||
# Rate limiting is nearly always downstream: retries against a hostname with no DNS record
|
||||
# exhaust the allowance, which then fails renewals for domains that have nothing wrong with
|
||||
# them. Reporting the rate limit as the problem sends the operator to fix the wrong domains.
|
||||
#
|
||||
# Rotation suffix is the clock, not mtime.
|
||||
# Every file in this directory carries the same mtime because they arrive as a synced set, so
|
||||
# ordering by mtime returns an arbitrary answer that looks authoritative.
|
||||
#
|
||||
# Bounded by design, not by hope.
|
||||
# 639 MB across 1001 files cannot be read in a page load. Only CERT_TRIAGE_FILES logs are
|
||||
# opened, and only CERT_TRIAGE_MAX_BYTES from the end of each, because a failure explains
|
||||
# itself at the end of the run rather than the start.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only throughout. It opens certbot's logs and nothing else — no certificate is requested,
|
||||
# renewed or deleted, and no log is rotated or truncated by anything here.
|
||||
#
|
||||
# Diagnosis only. This names why renewals failed; acting on that is the operator's, and
|
||||
# cert_history.sh remains the thing that records what happened.
|
||||
#
|
||||
# The log directory is normally discovered from the NPM container rather than hardcoded, so a
|
||||
# container path change surfaces as "no logs found" rather than as a silently empty triage.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# cert_triage.sh summary — categories, affected domains, and the causal reading
|
||||
# cert_triage.sh --json the same as JSON, for the Certs tab
|
||||
# cert_triage.sh --files=N override how many rotated logs to read
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# CERT_TRIAGE_FILES rotated logs to read, newest first
|
||||
# CERT_TRIAGE_MAX_BYTES bytes read from the end of each
|
||||
# CERT_TRIAGE_LOG_DIR override the log directory; normally found from the NPM container
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/cert_triage.php" "$@"
|
||||
@@ -0,0 +1,233 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Checks how the settings UI will draw every conf field, and proves the inference rules against
|
||||
// the conventions the conf files actually use. Two modes in one tool: fixed assertions over
|
||||
// synthetic snippets, and a report of what this host's real master.conf infers to.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Hand-run, reads only, writes nothing — the same shape as Tools/ai_log_check.sh. Run it after
|
||||
// touching _vv_conf_widget(), after adding a conf convention, or when a setting draws as the
|
||||
// wrong control and the question is whether the rule or the comment above it is at fault.
|
||||
//
|
||||
// php Tools/conf_widget_check.php assertions, then the live summary
|
||||
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Assertions run against snippets, never against the live conf.
|
||||
// The live conf is the thing being described, so it cannot also be the thing that proves
|
||||
// the description right — an inference rule that silently stopped matching would keep
|
||||
// passing as the conf drifted to suit it. The snippets are frozen copies of each
|
||||
// convention as written, so a rule change that breaks one shows up here rather than as a
|
||||
// wrong control on a page.
|
||||
//
|
||||
// An inference is a drawing decision, not a validation.
|
||||
// Consistent with confform.php, none of this validates. A number field carrying min and
|
||||
// max is a courtesy to whoever is typing, not a promise the value is sensible — the
|
||||
// consuming script still owns that question.
|
||||
//
|
||||
// The live pass reports, it does not assert.
|
||||
// What this host's master.conf infers to is a description of that file, not a verdict on
|
||||
// it. Turning the live summary into pass/fail would make an unusual but legitimate
|
||||
// setting look like a defect.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Parses conf and reports; writes no conf, no store and no page.
|
||||
//
|
||||
// Exits non-zero only when a snippet assertion fails, so it can gate a commit without a real
|
||||
// conf's contents ever being able to break the build.
|
||||
//
|
||||
// Never renders. It reports which control would be drawn; the drawing stays in confform.php,
|
||||
// so this cannot disagree with the page by construction.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php Tools/conf_widget_check.php assertions, then the live summary
|
||||
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
||||
//
|
||||
// Hand-run. Not scheduled and in no orchestrator — run it after touching _vv_conf_widget(),
|
||||
// after adding a conf convention, or when a setting draws as the wrong control.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/confform.php _vv_conf_parse_field_range(), vv_conf_key_is_secret()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once __DIR__ . '/../include/confform.php';
|
||||
|
||||
$pass = 0; $fail = 0; $failed = [];
|
||||
function check(string $name, $got, $want): void {
|
||||
global $pass, $fail, $failed;
|
||||
if ($got === $want) { $pass++; return; }
|
||||
$fail++;
|
||||
$failed[] = sprintf("%-42s got %s want %s", $name, json_encode($got), json_encode($want));
|
||||
}
|
||||
|
||||
// Parse one snippet and return its fields keyed by name.
|
||||
function vv_wc_fields(string $snippet): array {
|
||||
$lines = explode("\n", $snippet);
|
||||
$out = [];
|
||||
foreach (_vv_conf_parse_field_range($lines, 0, count($lines), 'master.conf') as $f) {
|
||||
$out[$f['key']] = $f;
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
|
||||
// ── Booleans and numbers ─────────────────────────────────────────────────────────────────────
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
AI_ENABLED=true
|
||||
AI_OFF=false
|
||||
AI_QUOTED_BOOL="true"
|
||||
AI_CONNECT_TIMEOUT=5 # seconds — probe when resolving which node has Ollama
|
||||
AI_CHAT_HISTORY_MAX=10 # conversations kept — clamped to 1-50
|
||||
AI_PLAIN_NUMBER=32
|
||||
AI_NEGATIVE=-1
|
||||
CONF);
|
||||
check('bool true', $f['AI_ENABLED']['widget'], 'bool');
|
||||
check('bool false', $f['AI_OFF']['widget'], 'bool');
|
||||
check('bool quoted', $f['AI_QUOTED_BOOL']['widget'], 'bool');
|
||||
check('int', $f['AI_CONNECT_TIMEOUT']['widget'], 'int');
|
||||
check('int unit', $f['AI_CONNECT_TIMEOUT']['unit'] ?? null, 'seconds');
|
||||
check('int without a unit', $f['AI_PLAIN_NUMBER']['unit'] ?? null, null);
|
||||
check('int range min', $f['AI_CHAT_HISTORY_MAX']['min'] ?? null, 1);
|
||||
check('int range max', $f['AI_CHAT_HISTORY_MAX']['max'] ?? null, 50);
|
||||
check('int range keeps unit', $f['AI_CHAT_HISTORY_MAX']['unit'] ?? null, 'conversations');
|
||||
check('negative int', $f['AI_NEGATIVE']['widget'], 'int');
|
||||
|
||||
// ── File modes must never become spinners ────────────────────────────────────────────────────
|
||||
// A spinner invites arrowing 755 to 756, and a number input normalises a leading zero away.
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
PERMISSIONS_DIR_MODE="755" # directories
|
||||
PERMISSIONS_FILE_MODE="664" # files
|
||||
TRANSCODE_CHMOD="755"
|
||||
LEADING_ZERO_MODE="0755"
|
||||
TRANSCODE_MANAGER_MODE="smart"
|
||||
SOME_TIMEOUT_MODE=30 # seconds
|
||||
CONF);
|
||||
check('dir mode is text', $f['PERMISSIONS_DIR_MODE']['widget'], 'text');
|
||||
check('file mode is text', $f['PERMISSIONS_FILE_MODE']['widget'], 'text');
|
||||
check('chmod is text', $f['TRANSCODE_CHMOD']['widget'], 'text');
|
||||
check('octal mode is text', $f['LEADING_ZERO_MODE']['widget'], 'text');
|
||||
check('worded mode is text', $f['TRANSCODE_MANAGER_MODE']['widget'], 'text');
|
||||
// The guard is about octal, not about the word MODE — a real duration is still a number.
|
||||
check('non-octal MODE is int', $f['SOME_TIMEOUT_MODE']['widget'], 'int');
|
||||
|
||||
// ── Enum, quoted-line convention ─────────────────────────────────────────────────────────────
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
# Controls behaviour when versions differ.
|
||||
# "warn" — log warning and continue
|
||||
# "abort" — refuse to continue (strict)
|
||||
UNRAID_VERSION_MISMATCH_ACTION="warn"
|
||||
CONF);
|
||||
check('quoted-line enum', $f['UNRAID_VERSION_MISMATCH_ACTION']['widget'], 'enum');
|
||||
check('enum choices', array_column($f['UNRAID_VERSION_MISMATCH_ACTION']['choices'], 'value'),
|
||||
['warn', 'abort']);
|
||||
check('enum keeps its hints', $f['UNRAID_VERSION_MISMATCH_ACTION']['choices'][1]['hint'],
|
||||
'refuse to continue (strict)');
|
||||
|
||||
// One quoted comment line is a quotation, not a choice list of one.
|
||||
$f = vv_wc_fields("# \"off\" means something here\n SOMETHING=\"off\"");
|
||||
check('lone quoted line', $f['SOMETHING']['widget'], 'text');
|
||||
|
||||
// A value outside its own documented list is surfaced, never dropped to tidy the list — dropping
|
||||
// it would change the setting the moment the form saved.
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
# "warn" — log and continue
|
||||
# "abort" — refuse
|
||||
DRIFTED="something_else"
|
||||
CONF);
|
||||
check('drifted value stays enum', $f['DRIFTED']['widget'], 'enum');
|
||||
check('drifted value offered', $f['DRIFTED']['choices'][0]['value'], 'something_else');
|
||||
check('drifted list intact', count($f['DRIFTED']['choices']), 3);
|
||||
|
||||
// ── Enum, inline pipe convention ─────────────────────────────────────────────────────────────
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
DIGEST_PROFILE="weekly" # always | smart | weekly
|
||||
SONARR_MONITOR="all" # monitor mode on add: all | future | first | none
|
||||
PROSE_WITH_PIPE="x" # pipe the output through grep | sort
|
||||
CONF);
|
||||
check('inline enum', $f['DIGEST_PROFILE']['widget'], 'enum');
|
||||
check('inline enum choices', array_column($f['DIGEST_PROFILE']['choices'], 'value'),
|
||||
['always', 'smart', 'weekly']);
|
||||
check('inline enum after colon',$f['SONARR_MONITOR']['widget'], 'enum');
|
||||
// The value is not among them, so this is prose that happens to contain a pipe.
|
||||
check('prose with a pipe', $f['PROSE_WITH_PIPE']['widget'], 'text');
|
||||
|
||||
// ── Secrets ──────────────────────────────────────────────────────────────────────────────────
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
HOST1_NPM_PASS="hunter2"
|
||||
HOST1_SONARR_API_KEY="abcdef"
|
||||
WEBHOOK_SECRET="s3cr3t"
|
||||
AI_TOKEN_DB="${AI_DATA_DIR}/ai_token_history.db"
|
||||
AI_TOKEN_RETAIN_ROWS=20000
|
||||
AI_TOKEN_SYNC_ENABLED=true
|
||||
CONF);
|
||||
check('password masked', $f['HOST1_NPM_PASS']['widget'], 'secret');
|
||||
check('api key masked', $f['HOST1_SONARR_API_KEY']['widget'], 'secret');
|
||||
check('secret masked', $f['WEBHOOK_SECRET']['widget'], 'secret');
|
||||
// LLM token accounting is not a credential. Masking these was the bug this exception fixes.
|
||||
check('AI_TOKEN_DB visible', $f['AI_TOKEN_DB']['widget'], 'path');
|
||||
check('AI_TOKEN_RETAIN visible', $f['AI_TOKEN_RETAIN_ROWS']['widget'], 'int');
|
||||
check('AI_TOKEN_SYNC visible', $f['AI_TOKEN_SYNC_ENABLED']['widget'], 'bool');
|
||||
|
||||
// ── Paths, text, arrays ──────────────────────────────────────────────────────────────────────
|
||||
$f = vv_wc_fields(<<<'CONF'
|
||||
AI_INDEX_DB="${AI_DATA_DIR}/ai_index.db"
|
||||
SOME_ABS="/mnt/user/appdata"
|
||||
AI_MODEL="qwen3:14b"
|
||||
AI_PROFILES="*"
|
||||
AI_CONF_WRITE_KEYS=()
|
||||
LIST_MULTI=(
|
||||
one
|
||||
two
|
||||
)
|
||||
CONF);
|
||||
check('templated path', $f['AI_INDEX_DB']['widget'], 'path');
|
||||
check('absolute path', $f['SOME_ABS']['widget'], 'path');
|
||||
check('plain text', $f['AI_MODEL']['widget'], 'text');
|
||||
check('wildcard text', $f['AI_PROFILES']['widget'], 'text');
|
||||
check('single-line array', $f['AI_CONF_WRITE_KEYS']['widget'], 'lines');
|
||||
check('multi-line array', $f['LIST_MULTI']['widget'], 'lines');
|
||||
// The scratch field the parser uses to reach the comment lines must not ride out to the browser.
|
||||
check('no _lines leak', isset($f['AI_MODEL']['_lines']), false);
|
||||
|
||||
printf("assertions: %d passed, %d failed\n", $pass, $fail);
|
||||
foreach ($failed as $l) echo " FAIL $l\n";
|
||||
|
||||
// ── What this host's conf actually infers to ─────────────────────────────────────────────────
|
||||
$path = CONF_DIR . '/master.conf';
|
||||
if (!is_readable($path)) {
|
||||
echo "\nmaster.conf not readable at $path — skipping the live summary\n";
|
||||
exit($fail ? 1 : 0);
|
||||
}
|
||||
$lines = explode("\n", (string) file_get_contents($path));
|
||||
$hdrs = [];
|
||||
foreach ($lines as $i => $l) {
|
||||
if (preg_match('/^#\s*[━─]{2,}\s+([A-Za-z].+?)\s+[━─]{2,}/u', $l, $m)) $hdrs[] = [$i, trim($m[1])];
|
||||
}
|
||||
|
||||
$list = in_array('--list', $argv, true);
|
||||
$tally = []; $n = 0; $enums = []; $ranges = [];
|
||||
foreach ($hdrs as $k => [$i, $name]) {
|
||||
$end = $hdrs[$k + 1][0] ?? count($lines);
|
||||
$rows = _vv_conf_parse_field_range($lines, $i + 1, $end, 'master.conf');
|
||||
if ($list && $rows) printf("\n── %s\n", $name);
|
||||
foreach ($rows as $x) {
|
||||
$n++;
|
||||
$w = $x['widget'] ?? '?';
|
||||
$tally[$w] = ($tally[$w] ?? 0) + 1;
|
||||
if ($w === 'enum') $enums[] = $x['key'] . ' = ' . implode(' | ', array_column($x['choices'], 'value'));
|
||||
if (isset($x['min'])) $ranges[] = $x['key'] . ' ' . $x['min'] . '-' . $x['max'];
|
||||
if ($list) {
|
||||
$e = [];
|
||||
if (isset($x['unit'])) $e[] = $x['unit'];
|
||||
if (isset($x['min'])) $e[] = $x['min'] . '-' . $x['max'];
|
||||
printf(" %-34s %-7s %s\n", $x['key'], $w, $e ? '[' . implode(' ', $e) . ']' : '');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
printf("\nlive master.conf: %d fields across %d sections\n", $n, count($hdrs));
|
||||
ksort($tally);
|
||||
foreach ($tally as $k => $v) printf(" %-8s %d\n", $k, $v);
|
||||
if ($enums) { echo "\nchoice lists found:\n"; foreach ($enums as $e) echo " $e\n"; }
|
||||
if ($ranges) { echo "\nbounded numbers found:\n"; foreach ($ranges as $e) echo " $e\n"; }
|
||||
|
||||
exit($fail ? 1 : 0);
|
||||
Executable
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/php -q
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Creates or updates the per-partner folder in folder.view3 — "<PartnerShort>-Fallback" — that
|
||||
// holds the containers this host runs on the partner's behalf. Named from master.conf, so the
|
||||
// folder name follows the hostnames rather than a value someone has to keep in step by hand.
|
||||
//
|
||||
// Onboard calls this on the mirror after deploying the owner's stack, so those containers land
|
||||
// somewhere that says whose they are instead of scattering into the mirror's own groups.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Create-or-update, never replace. The named folder is found in folder.view3 if it exists and
|
||||
// is amended in place; only its absence causes one to be created. Every other folder in the
|
||||
// file, and every container already filed elsewhere, is left exactly as it was — this is one
|
||||
// shelf in someone else's cupboard.
|
||||
//
|
||||
// Called during onboard on the mirror, after the owner's stack has been deployed there, so the
|
||||
// containers exist by the time anything tries to file them.
|
||||
//
|
||||
// The icon is resolved separately from the folder and never gates it. --icon-only performs
|
||||
// just that lookup and prints the URL, which is how onboard asks for it without writing
|
||||
// anything.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The name comes from master.conf, never from a second source.
|
||||
// HOST2="unRAID-Jayred36" → "Jayred36-Fallback". The convention already existed by hand as
|
||||
// "Jayred365-Fallback", which did not match any derivable value — so the folder is renamed
|
||||
// to follow the conf rather than a conf field being invented to follow the folder.
|
||||
//
|
||||
// Upsert by name, never append.
|
||||
// folder.view3 keys folders by a random id, so writing without looking produces a second
|
||||
// folder with the same name and half the contents. That is exactly what happened here: a
|
||||
// "Jayred365-Fallback" with the icon and no containers, beside one with the containers and
|
||||
// no icon. Match on name, keep the existing id.
|
||||
//
|
||||
// This writes one folder, it does not replace the file.
|
||||
// include/docker.php used to mirror Varaverk's whole folder store over the top of
|
||||
// folder.view3's, destroying anything created in that plugin's UI. That write is gone. A
|
||||
// co-writer that edits a single key it owns is a different thing from one that overwrites
|
||||
// everything, and only the second kind loses data.
|
||||
//
|
||||
// Closest Emby user wins the icon, and a wrong guess costs nothing.
|
||||
// The folder image is decoration. Exact match first, then lowest Levenshtein distance
|
||||
// within a bound. "Jayred36" resolves to "Jayred365" (distance 1) over "Jayred" (2).
|
||||
// This is deliberately unlike resolve_tailscale_ip's exact-prefix-plus-ambiguity-guard —
|
||||
// there, a wrong match sends data to the wrong machine; here it picks the wrong avatar.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Atomic write — .vv.tmp then rename(), so folder.view3 never reads a truncated file.
|
||||
// Absent plugin is a clean skip, not an error — nothing to do if folder.view3 is not installed.
|
||||
// --dry-run prints the resulting folder and writes nothing.
|
||||
// Icon resolution failing never blocks the folder: no image is a cosmetic loss, no folder is not.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// fallback_folder.php --host=HOST2 [--containers=A,B,C] [--icon=URL] [--dry-run]
|
||||
// fallback_folder.php --host=HOST2 --icon-only resolve and print the icon URL, write nothing
|
||||
//
|
||||
// CONFIGURATION
|
||||
// HOST1/HOST2… master.conf — the hostname the folder is named after
|
||||
// HOST*_EMBY_URL where to look users up (local is fine, it is a server-side call)
|
||||
// HOST*_EMBY_API_KEY "
|
||||
// HOST*_EMBY_PUBLIC_URL base the ICON is built from. Must be reachable from a browser on
|
||||
// either host, so localhost:8096 is not it — the icon renders in the
|
||||
// WebGUI of whichever machine is looking. Empty means no icon.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
$pluginDir = dirname(__DIR__);
|
||||
require_once $pluginDir . '/include/config.php';
|
||||
|
||||
define('FV3_JSON', '/boot/config/plugins/folder.view3/docker.json');
|
||||
|
||||
// ── Args ──────────────────────────────────────────────────────────────────────
|
||||
$opts = ['host' => '', 'containers' => '', 'icon' => '', 'dry-run' => false, 'icon-only' => false];
|
||||
foreach (array_slice($argv, 1) as $a) {
|
||||
if ($a === '--dry-run') { $opts['dry-run'] = true; continue; }
|
||||
if ($a === '--icon-only'){ $opts['icon-only'] = true; continue; }
|
||||
if (preg_match('/^--([a-z-]+)=(.*)$/', $a, $m)) $opts[$m[1]] = $m[2];
|
||||
}
|
||||
$hostId = strtoupper(trim($opts['host']));
|
||||
if (!preg_match('/^HOST\d+$/', $hostId)) {
|
||||
fwrite(STDERR, "usage: fallback_folder.php --host=HOST2 [--containers=A,B] [--dry-run]\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
$vars = vv_conf_vars();
|
||||
$hostname = trim($vars[$hostId] ?? '');
|
||||
if ($hostname === '') { fwrite(STDERR, "$hostId is not set in master.conf\n"); exit(1); }
|
||||
|
||||
|
||||
// derive_short_name() in common.sh: lowercase, strip a leading "unraid-", capitalise.
|
||||
$short = preg_replace('/^unraid-/i', '', $hostname);
|
||||
$short = ucfirst($short);
|
||||
$folderName = $short . '-Fallback';
|
||||
|
||||
// ── Icon: closest Emby user ───────────────────────────────────────────────────
|
||||
function vv_closest_emby_icon(string $short, array $vars, string $meId): array {
|
||||
$url = rtrim(trim($vars[$meId . '_EMBY_URL'] ?? ''), '/');
|
||||
$key = trim($vars[$meId . '_EMBY_API_KEY'] ?? '');
|
||||
$pub = rtrim(trim($vars[$meId . '_EMBY_PUBLIC_URL'] ?? ''), '/');
|
||||
if ($url === '' || $key === '') return ['', 'no Emby url/key configured'];
|
||||
|
||||
$raw = @file_get_contents("$url/Users?api_key=" . urlencode($key), false,
|
||||
stream_context_create(['http' => ['timeout' => 10]]));
|
||||
$users = json_decode((string)$raw, true);
|
||||
if (!is_array($users) || !$users) return ['', 'Emby returned no users'];
|
||||
|
||||
$needle = strtolower($short);
|
||||
$best = null; $bestD = PHP_INT_MAX; $runnerUp = null;
|
||||
foreach ($users as $u) {
|
||||
$name = (string)($u['Name'] ?? '');
|
||||
if ($name === '' || empty($u['Id'])) continue;
|
||||
$d = levenshtein($needle, strtolower($name));
|
||||
if ($d < $bestD) { $runnerUp = $best; $bestD = $d; $best = $u; }
|
||||
elseif ($runnerUp === null || $d < levenshtein($needle, strtolower((string)$runnerUp['Name']))) {
|
||||
$runnerUp = $u;
|
||||
}
|
||||
}
|
||||
if (!$best) return ['', 'no candidate users'];
|
||||
|
||||
// Bound it so an unrelated name never wins by being the least-bad of a bad field.
|
||||
$limit = max(2, (int)floor(strlen($needle) * 0.5));
|
||||
if ($bestD > $limit) {
|
||||
return ['', sprintf('closest was "%s" (distance %d > limit %d) — too far, no icon',
|
||||
$best['Name'], $bestD, $limit)];
|
||||
}
|
||||
if ($pub === '') {
|
||||
return ['', sprintf('matched Emby user "%s" (distance %d) but %s_EMBY_PUBLIC_URL is unset — '
|
||||
. 'a localhost icon would not render in a browser', $best['Name'], $bestD, $meId)];
|
||||
}
|
||||
$tag = (string)($best['PrimaryImageTag'] ?? '');
|
||||
if ($tag === '') return ['', sprintf('Emby user "%s" has no primary image', $best['Name'])];
|
||||
|
||||
$icon = sprintf('%s/Users/%s/Images/Primary?maxWidth=200&tag=%s&quality=90',
|
||||
$pub, $best['Id'], $tag);
|
||||
$note = sprintf('matched Emby user "%s" (distance %d%s)', $best['Name'], $bestD,
|
||||
$runnerUp ? sprintf(', next "%s" at %d', $runnerUp['Name'],
|
||||
levenshtein($needle, strtolower((string)$runnerUp['Name']))) : '');
|
||||
return [$icon, $note];
|
||||
}
|
||||
|
||||
$meId = strtoupper(vv_detect_host());
|
||||
|
||||
// --icon= skips the lookup entirely. The mirror needs this: it is being given a folder named
|
||||
// after the OWNER, and the avatar lives in the owner's Emby — which the mirror has no key for and
|
||||
// may not run at all. So the owner resolves the URL with --icon-only and hands it over, rather
|
||||
// than the mirror guessing from a userbase it cannot see.
|
||||
if (trim($opts['icon']) !== '') {
|
||||
$icon = trim($opts['icon']);
|
||||
$iconNote = 'supplied by caller';
|
||||
} else {
|
||||
[$icon, $iconNote] = vv_closest_emby_icon($short, $vars, $meId);
|
||||
}
|
||||
|
||||
// --icon-only: resolve and print, touch nothing. Exits non-zero when there is no icon, so a
|
||||
// caller can tell "no image" from "empty string because something broke".
|
||||
if ($opts['icon-only']) {
|
||||
if ($icon === '') { fwrite(STDERR, "no icon: $iconNote\n"); exit(1); }
|
||||
echo $icon . "\n";
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// ── Upsert the folder ─────────────────────────────────────────────────────────
|
||||
// A host does not keep a fallback folder for itself — the folder means "containers I run on
|
||||
// SOMEONE ELSE's behalf", so naming it after this machine is always wrong. Checked here rather
|
||||
// than at argument parsing, because --icon-only legitimately asks for THIS host's own avatar:
|
||||
// the owner resolves its own picture to hand to the mirror, which is the whole point of that mode.
|
||||
//
|
||||
// Refused rather than created, because the failure is otherwise silent — an empty folder named
|
||||
// after yourself looks plausible enough to survive a glance. One appeared on HOST1 exactly this
|
||||
// way, when an older copy of this script ignored an unrecognised flag and ran the upsert anyway.
|
||||
if (strcasecmp($hostId, strtoupper(vv_detect_host())) === 0) {
|
||||
fwrite(STDERR, "$hostId is this host — a fallback folder is named after the PARTNER, not self\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
if (!file_exists(FV3_JSON)) {
|
||||
echo "folder.view3 is not installed — nothing to do\n";
|
||||
exit(0);
|
||||
}
|
||||
$j = json_decode((string)@file_get_contents(FV3_JSON), true);
|
||||
if (!is_array($j)) { fwrite(STDERR, "folder.view3 docker.json is unreadable\n"); exit(1); }
|
||||
|
||||
$containers = array_values(array_filter(array_map('trim', explode(',', (string)$opts['containers']))));
|
||||
|
||||
// Match on name, case-insensitively, so a hand-made folder is adopted rather than duplicated.
|
||||
$targetId = null;
|
||||
foreach ($j as $id => $f) {
|
||||
if (is_array($f) && strcasecmp((string)($f['name'] ?? ''), $folderName) === 0) { $targetId = $id; break; }
|
||||
}
|
||||
$created = false;
|
||||
if ($targetId === null) {
|
||||
// folder.view3's own id shape: 20 chars of url-safe base64.
|
||||
$targetId = substr(str_replace(['+', '/', '='], '', base64_encode(random_bytes(15))), 0, 20);
|
||||
$j[$targetId] = ['name' => $folderName, 'icon' => '', 'settings' => ['', '', '1', '', '1', ''],
|
||||
'regex' => '', 'containers' => [], 'containerImages' => []];
|
||||
$created = true;
|
||||
}
|
||||
$j[$targetId]['name'] = $folderName;
|
||||
if ($icon !== '') $j[$targetId]['icon'] = $icon;
|
||||
foreach ($containers as $c) {
|
||||
if (!in_array($c, (array)$j[$targetId]['containers'], true)) $j[$targetId]['containers'][] = $c;
|
||||
}
|
||||
$j[$targetId]['containers'] = array_values((array)$j[$targetId]['containers']);
|
||||
|
||||
printf("%s %s (id %s)\n", $created ? 'create' : 'update', $folderName, $targetId);
|
||||
printf(" icon : %s\n", $icon !== '' ? $icon : '(none) — ' . $iconNote);
|
||||
if ($icon !== '') printf(" via : %s\n", $iconNote);
|
||||
printf(" containers: %s\n", implode(', ', (array)$j[$targetId]['containers']) ?: '(none)');
|
||||
|
||||
if ($opts['dry-run']) { echo " DRY RUN — nothing written\n"; exit(0); }
|
||||
|
||||
$tmp = FV3_JSON . '.vv.tmp';
|
||||
if (file_put_contents($tmp, json_encode($j, JSON_UNESCAPED_SLASHES)) === false || !rename($tmp, FV3_JSON)) {
|
||||
fwrite(STDERR, "failed to write " . FV3_JSON . "\n"); exit(1);
|
||||
}
|
||||
echo " written ✅\n";
|
||||
Executable
+337
@@ -0,0 +1,337 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= WebGUI JavaScript Checker ======================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ==============================================================================================
|
||||
# Finds the two JavaScript faults this plugin has actually shipped, neither of which any syntax
|
||||
# check can see, because both are runtime behaviour:
|
||||
#
|
||||
# 1. An identifier declared inside one function and referenced from another. Throws
|
||||
# ReferenceError on every call, kills the rest of the render, and passes php -l and
|
||||
# node --check without complaint.
|
||||
#
|
||||
# 2. A fetch chain ending in an empty catch. Not error handling — error deletion. The request
|
||||
# fails, nothing renders, nothing is logged, and the surface sits on "Loading…" forever.
|
||||
#
|
||||
# The two compound: on 2026-08-20 a cross-scope ReferenceError in the mesh chat was swallowed by
|
||||
# an empty catch on every render. It presented as "the chat takes a minute to load" — the minute
|
||||
# was the poller's backoff — and hours went into profiling PHP that was never slow. Once a catch
|
||||
# reported the error, the fault named itself in one line.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# 1. Extract the JS from each <script> block in a PHP/page file
|
||||
# 2. Strip PHP tags, comments, string literals, template-literal text and object keys
|
||||
# 3. Cross-scope pass — declarations per function, then uses judged against them
|
||||
# 4. Catch pass — an empty .catch() within 25 lines of a fetch()
|
||||
#
|
||||
# Reads only. Prints findings and exits non-zero when any are found, so it can gate a commit.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A Checker That Cannot Fail Loudly Is Worse Than No Checker
|
||||
# --self-test runs both detectors against a fixture carrying both faults and asserts they are
|
||||
# found. It is not decoration. The first version of this scan reported a confident zero across
|
||||
# the whole plugin because a regex was silently broken; the fixture is what catches that.
|
||||
#
|
||||
# Preprocessing Is Where The False Positives Die
|
||||
# A raw scan produced ~1600 candidates, nearly all of them HTML attribute names, CSS keywords
|
||||
# and English prose living inside template literals. Keeping only the ${...} expressions cut
|
||||
# that to double digits. What remains after preprocessing is worth a human's attention.
|
||||
#
|
||||
# Report, Never Rewrite
|
||||
# This prints file and line. It does not edit. A tool that silently "fixes" a false positive in
|
||||
# a render path is a worse outcome than the fault it was hunting.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Escaped $ In Every Perl Character Class
|
||||
# [A-Za-z_$] makes Perl interpolate $] — its version variable — and silently mangles the
|
||||
# pattern into something that matches nothing. That mistake produced a false clean result twice
|
||||
# while this was being written. Every class here writes \$, and --self-test would catch it
|
||||
# returning.
|
||||
#
|
||||
# Known Parser Gaps, Stated Rather Than Hidden
|
||||
# This is regex and brace counting, not a JavaScript parser. It over-reports and never
|
||||
# under-reports, which is the safe direction: a clean run is meaningful, a dirty one needs a
|
||||
# human. Preprocessing took the raw count from ~1600 to single digits by handling template
|
||||
# literals, string literals, object keys, regex literals, destructuring, nested-function
|
||||
# parameters and multi-declarator const.
|
||||
#
|
||||
# BASELINE as of 2026-08-21: three known false positives on a healthy tree —
|
||||
# setup.php `to`, `id` in vvRenderOnboardPanel()
|
||||
# Varaverk.page `s` in vvRenderMirrorOnboard()
|
||||
# Each was read and confirmed harmless. Findings beyond those three are new and worth opening.
|
||||
# If that baseline ever reaches zero, delete this paragraph rather than letting it rot.
|
||||
#
|
||||
# Non-Zero Exit On Findings
|
||||
# So it can sit in front of a commit. A checker whose output has to be noticed by a human is a
|
||||
# checker that stops being run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# js_check.sh
|
||||
# Scan every pages/*.php, include/*.php and Varaverk.page.
|
||||
#
|
||||
# js_check.sh --scope | --catches
|
||||
# One pass only.
|
||||
#
|
||||
# js_check.sh --self-test
|
||||
# Verify both detectors against the built-in fixture. Run this after editing this file.
|
||||
#
|
||||
# js_check.sh <file> [...]
|
||||
# Scan named files instead of the whole plugin.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
DO_SCOPE=true; DO_CATCH=true; SELF_TEST=false
|
||||
FILES=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--scope) DO_CATCH=false ;;
|
||||
--catches) DO_SCOPE=false ;;
|
||||
--self-test) SELF_TEST=true ;;
|
||||
-*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||
*) FILES+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Preprocessor ──────────────────────────────────────────────────────────────────────────────
|
||||
cat > "$WORK/prep.pl" <<'PREP'
|
||||
# NOTE: every character class escapes $ as \$. Bare [A-Za-z_$] interpolates $] (Perl's version)
|
||||
# and silently matches nothing — see OPERATIONAL SAFEGUARDS.
|
||||
local $/; my $s = <>;
|
||||
$s =~ s{<\?=.*?\?>}{0}gs;
|
||||
$s =~ s{<\?php.*?\?>}{}gs;
|
||||
$s =~ s{/\*.*?\*/}{}gs;
|
||||
$s =~ s{//[^\n]*}{}g;
|
||||
# Template literals carry HTML, CSS and prose. Only the ${...} expressions are code.
|
||||
$s =~ s{`((?:[^`\\]|\\.)*)`}{ " " . join(" ", $1 =~ m{\$\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}}g) . " " }ges;
|
||||
$s =~ s{'(?:[^'\\\n]|\\.)*'}{""}g;
|
||||
$s =~ s{"(?:[^"\\\n]|\\.)*"}{""}g;
|
||||
# Regex literals. Only where one can legally begin — after ( = , : return — so division is not
|
||||
# mistaken for a pattern. Their contents and flags are not identifiers: /\s*os|windows/i was
|
||||
# reporting s, os and windows as undeclared variables.
|
||||
$s =~ s{(=>\s*|[=(,:\[;?!]\s*|[&|]{2}\s*|\breturn\s+)/(?:[^/\\\n]|\\.)+/[gimsuy]*}{$1 0}g;
|
||||
# Object-literal keys are not references. Anchored to { or , so a ternary keeps its operand.
|
||||
$s =~ s/([{,]\s*)[A-Za-z_\$][A-Za-z0-9_\$]*\s*:/$1 /g;
|
||||
print $s;
|
||||
PREP
|
||||
|
||||
# ── Cross-scope detector ──────────────────────────────────────────────────────────────────────
|
||||
cat > "$WORK/scope.awk" <<'SCOPE'
|
||||
function fname(l, m) { if (match(l, /function[ \t]+[A-Za-z_$][A-Za-z0-9_$]*/)) { m=substr(l,RSTART,RLENGTH); sub(/function[ \t]+/,"",m); return m } return "" }
|
||||
BEGIN {
|
||||
# Built-ins and browser globals. Belt and braces — a parser slip that registers one of these as
|
||||
# a local would otherwise report it in every other function in the file, which is exactly how
|
||||
# Math appeared fifteen times while this was being written.
|
||||
split("Math JSON Object Array String Number Boolean Date RegExp Error Promise Set Map WeakMap " \
|
||||
"Symbol BigInt Intl console document window navigator location history screen localStorage " \
|
||||
"sessionStorage fetch setTimeout setInterval clearTimeout clearInterval requestAnimationFrame " \
|
||||
"parseInt parseFloat isNaN isFinite encodeURIComponent decodeURIComponent URLSearchParams " \
|
||||
"FormData Headers Request Response AbortController CustomEvent Event Node Element " \
|
||||
"getComputedStyle matchMedia structuredClone queueMicrotask btoa atob crypto performance " \
|
||||
"globalThis undefined NaN Infinity ResizeObserver MutationObserver Uint32Array swal", g, " ")
|
||||
for (gi in g) glob[g[gi]]=1
|
||||
}
|
||||
{ L[NR]=$0 }
|
||||
END {
|
||||
depth=0; cur=""
|
||||
for (n=1; n<=NR; n++) {
|
||||
line=L[n]
|
||||
if (depth==0) { f=fname(line); if (f!="") cur=f }
|
||||
scope = (cur=="" ? "(module)" : cur)
|
||||
# Parameters of ANY function on this line, nested helpers included. Attributing a nested
|
||||
# helper's params to the enclosing function is an approximation, and the right one: it can
|
||||
# only suppress a report, never invent one, and nested params were the largest false-positive
|
||||
# class in this codebase.
|
||||
if (match(line, /function[ \t]*[A-Za-z0-9_$]*[ \t]*\([^)]*\)/)) {
|
||||
pp=substr(line,RSTART,RLENGTH); sub(/.*\(/,"",pp); sub(/\).*/,"",pp)
|
||||
c=split(pp,pa,","); for (k=1;k<=c;k++) { gsub(/[ \t]/,"",pa[k]); sub(/=.*/,"",pa[k]); if (pa[k]!="") decl[scope,pa[k]]=n } }
|
||||
# Every declarator in the statement, not just the first: `const s = a, n = b` declares both,
|
||||
# and reading only `s` left `n` looking undeclared wherever it was used.
|
||||
if (match(line, /(const|let|var)[ \t]+/)) {
|
||||
rest=substr(line,RSTART+RLENGTH)
|
||||
# Destructuring binds every name inside the brackets: `for (const [folder, items] of …)`
|
||||
# and `const { a, b } = obj`. Handled before the comma split, which cannot read them.
|
||||
if (match(rest, /^[ \t]*[\[{][^\]}]*[\]}]/)) {
|
||||
db=substr(rest,RSTART,RLENGTH)
|
||||
while (match(db, /[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||
nm2=substr(db,RSTART,RLENGTH)
|
||||
decl[scope,nm2]=n; if (scope=="(module)") mod[nm2]=n
|
||||
db=substr(db,RSTART+RLENGTH) }
|
||||
}
|
||||
cc=split(rest,dl,",")
|
||||
for (kk=1;kk<=cc;kk++) {
|
||||
# The identifier must be followed by "=" or end the segment. Without that test a comma
|
||||
# inside the initialiser — `const pct = Math.min(a, Math.round(b))` — makes the second
|
||||
# argument look like a second declarator, and Math gets registered as a local.
|
||||
if (match(dl[kk], /^[ \t]*[A-Za-z_$][A-Za-z0-9_$]*[ \t]*(=[^=]|=$|$)/)) {
|
||||
nm=dl[kk]; sub(/^[ \t]*/,"",nm); sub(/[ \t]*=.*$/,"",nm); gsub(/[ \t]/,"",nm)
|
||||
if (nm!="") { decl[scope,nm]=n; if (scope=="(module)") mod[nm]=n }
|
||||
} else break # stop at the first non-declarator; the rest is an expression
|
||||
}
|
||||
}
|
||||
tmp=line
|
||||
while (match(tmp, /(const|let|var)[ \t]+[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||
d=substr(tmp,RSTART,RLENGTH); sub(/(const|let|var)[ \t]+/,"",d)
|
||||
decl[scope,d]=n; if (scope=="(module)") mod[d]=n
|
||||
tmp=substr(tmp,RSTART+RLENGTH) }
|
||||
tmp=line
|
||||
while (match(tmp, /\([^)]*\)[ \t]*=>/)) {
|
||||
d=substr(tmp,RSTART,RLENGTH); gsub(/[()=>\t]/," ",d); gsub(/[{}\[\]]/," ",d)
|
||||
c2=split(d,ap,","); for (k2=1;k2<=c2;k2++) { gsub(/[ \t]/,"",ap[k2]); sub(/=.*/,"",ap[k2]); if (ap[k2]!="") decl[scope,ap[k2]]=n }
|
||||
tmp=substr(tmp,RSTART+RLENGTH) }
|
||||
tmp=line
|
||||
while (match(tmp, /[A-Za-z_$][A-Za-z0-9_$]*[ \t]*=>/)) {
|
||||
d=substr(tmp,RSTART,RLENGTH); gsub(/[ \t=>]/,"",d); if (d!="") decl[scope,d]=n
|
||||
tmp=substr(tmp,RSTART+RLENGTH) }
|
||||
if (match(line, /for[ \t]*\([ \t]*(const|let|var)[ \t]+[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||
d=substr(line,RSTART,RLENGTH); sub(/.*[ \t]/,"",d); decl[scope,d]=n }
|
||||
if (match(line, /catch[ \t]*\([ \t]*[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||
d=substr(line,RSTART,RLENGTH); sub(/.*\([ \t]*/,"",d); decl[scope,d]=n }
|
||||
if (fname(line)!="") isfn[fname(line)]=1
|
||||
for (i=1;i<=length(line);i++) { c=substr(line,i,1); if (c=="{") depth++; else if (c=="}") { depth--; if (depth<=0) { depth=0; cur="" } } }
|
||||
}
|
||||
depth=0; cur=""
|
||||
for (n=1; n<=NR; n++) {
|
||||
line=L[n]
|
||||
if (depth==0) { f=fname(line); if (f!="") cur=f }
|
||||
scope = (cur=="" ? "(module)" : cur)
|
||||
if (scope!="(module)") {
|
||||
tmp=line
|
||||
while (match(tmp, /[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||
id=substr(tmp,RSTART,RLENGTH); pre=substr(tmp,RSTART-1,1)
|
||||
tmp=substr(tmp,RSTART+RLENGTH)
|
||||
if (pre=="." || id in isfn || id in mod || id in glob) continue
|
||||
if ((scope,id) in decl) continue
|
||||
for (o in decl) { split(o,q,SUBSEP); if (q[2]==id && q[1]!=scope && q[1]!="(module)") {
|
||||
key=id SUBSEP scope; if (!(rep[key]++)) printf " line %d: %s used in %s() — declared only inside %s()\n", n, id, scope, q[1]; break } }
|
||||
}
|
||||
}
|
||||
for (i=1;i<=length(line);i++) { c=substr(line,i,1); if (c=="{") depth++; else if (c=="}") { depth--; if (depth<=0) { depth=0; cur="" } } }
|
||||
}
|
||||
}
|
||||
SCOPE
|
||||
|
||||
scan_scope() {
|
||||
local f="$1"
|
||||
awk '/<script/{p=1;next} /<\/script>/{p=0;print ""} p' "$f" | perl "$WORK/prep.pl" > "$WORK/x.js" 2>/dev/null
|
||||
awk -f "$WORK/scope.awk" "$WORK/x.js" 2>/dev/null
|
||||
}
|
||||
|
||||
# An empty catch within 25 lines of a fetch. Line distance rather than real chain parsing: the
|
||||
# chains here are short, and a bare `catch {}` far from any request is usually a deliberate
|
||||
# localStorage or execCommand guard, which this must not report.
|
||||
scan_catches() {
|
||||
local f="$1"
|
||||
awk '/fetch\(|XMLHttpRequest/ { inf=NR }
|
||||
/\.catch\(\s*\(\s*[a-z_]*\s*\)\s*=>\s*\{\s*\}\s*\)/ {
|
||||
if (inf && NR-inf <= 25) printf " line %d: fetch chain ends in an empty catch\n", NR }' "$f"
|
||||
}
|
||||
|
||||
# ── Self-test ─────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$SELF_TEST" == true ]]; then
|
||||
cat > "$WORK/fixture.php" <<'FIX'
|
||||
<script>
|
||||
function outer() {
|
||||
const paletteMap = { a: '#fff' };
|
||||
return paletteMap;
|
||||
}
|
||||
function styler(x) {
|
||||
// reads a const that lives in outer() — ReferenceError at runtime
|
||||
if (paletteMap[x]) return paletteMap[x];
|
||||
return '';
|
||||
}
|
||||
function loader() {
|
||||
fetch('/api/thing')
|
||||
.then(r => r.json())
|
||||
.then(d => { render(d); })
|
||||
.catch(() => {});
|
||||
}
|
||||
</script>
|
||||
FIX
|
||||
fails=0
|
||||
echo "── self-test ────────────────────────────────────────────────"
|
||||
if scan_scope "$WORK/fixture.php" | grep -q "paletteMap"; then
|
||||
echo " cross-scope detector PASS"
|
||||
else
|
||||
echo " cross-scope detector FAIL — known fault not reported"; fails=1
|
||||
fi
|
||||
if scan_catches "$WORK/fixture.php" | grep -q "empty catch"; then
|
||||
echo " empty-catch detector PASS"
|
||||
else
|
||||
echo " empty-catch detector FAIL — known fault not reported"; fails=1
|
||||
fi
|
||||
# A clean file must stay clean, or the detector is merely reporting everything.
|
||||
cat > "$WORK/clean.php" <<'CLN'
|
||||
<script>
|
||||
const paletteMap = { a: '#fff' };
|
||||
function styler(x) { return paletteMap[x] || ''; }
|
||||
function loader() {
|
||||
fetch('/api/thing').then(r => r.json()).then(d => { styler(d); })
|
||||
.catch(e => report('thing', e));
|
||||
}
|
||||
</script>
|
||||
CLN
|
||||
if [[ -z "$(scan_scope "$WORK/clean.php")$(scan_catches "$WORK/clean.php")" ]]; then
|
||||
echo " clean file stays clean PASS"
|
||||
else
|
||||
echo " clean file stays clean FAIL — false positive on correct code"; fails=1
|
||||
fi
|
||||
echo "─────────────────────────────────────────────────────────────"
|
||||
[[ "$fails" -eq 0 ]] && echo "self-test OK" || echo "SELF-TEST FAILED — do not trust a clean scan"
|
||||
exit "$fails"
|
||||
fi
|
||||
|
||||
# ── Scan ──────────────────────────────────────────────────────────────────────────────────────
|
||||
if [[ "${#FILES[@]}" -eq 0 ]]; then
|
||||
while IFS= read -r p; do FILES+=("$p"); done < <(
|
||||
find "$PLUGIN_DIR/pages" "$PLUGIN_DIR/include" -maxdepth 1 -name '*.php' 2>/dev/null
|
||||
[[ -f "$PLUGIN_DIR/Varaverk.page" ]] && echo "$PLUGIN_DIR/Varaverk.page"
|
||||
)
|
||||
fi
|
||||
|
||||
found=0
|
||||
for f in "${FILES[@]}"; do
|
||||
[[ -f "$f" ]] || continue
|
||||
grep -q "<script" "$f" 2>/dev/null || continue
|
||||
out=""
|
||||
[[ "$DO_SCOPE" == true ]] && out+="$(scan_scope "$f")"
|
||||
if [[ "$DO_CATCH" == true ]]; then
|
||||
c="$(scan_catches "$f")"
|
||||
[[ -n "$c" ]] && out+=$'\n'"$c"
|
||||
fi
|
||||
out="$(echo "$out" | sed '/^$/d')"
|
||||
if [[ -n "$out" ]]; then
|
||||
echo "### $(basename "$f")"
|
||||
echo "$out"
|
||||
found=$(( found + $(echo "$out" | grep -c '^ line') ))
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
if [[ "$found" -eq 0 ]]; then
|
||||
echo "clean — no cross-scope references, no silent fetch catches"
|
||||
else
|
||||
echo "$found finding(s). Cross-scope hits over-report: regex literals, nested-function"
|
||||
echo "parameters, destructured for-of and multi-declarator const each read as undeclared."
|
||||
echo "Read each before changing anything."
|
||||
fi
|
||||
exit $(( found > 0 ? 1 : 0 ))
|
||||
@@ -0,0 +1,127 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Sample per-peer Tailscale byte counters, once a minute, so the Partnership page can report
|
||||
// how much has actually moved between the servers in this mesh over a window.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Appends one line per mesh peer to data/db/mesh_traffic.db:
|
||||
//
|
||||
// epoch|peer-hostname|txbytes|rxbytes
|
||||
//
|
||||
// Called from Tools/api_cache_writer.sh, which already runs every minute. A window total is
|
||||
// then the difference between the newest sample and the oldest one still inside that window.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Measure the link, not the tool.
|
||||
// The previous card totalled rsync's own logs, so it could only ever describe rsync — and
|
||||
// said "no data moved" while SSH, the arr APIs, conf pushes and the Unraid API were all
|
||||
// using the same link. Tailscale counts the bytes on the wire, whatever sent them.
|
||||
//
|
||||
// Mesh peers only.
|
||||
// The tailnet holds phones and workstations. Filtered against the HOST* hostnames in
|
||||
// master.conf so this measures the partnership, not the tailnet.
|
||||
//
|
||||
// Absolute counters are stored, never deltas.
|
||||
// A delta computed at write time bakes in whatever the sampling interval happened to be
|
||||
// and cannot be re-derived if a run is missed. Storing the raw counter means a gap costs
|
||||
// resolution, not correctness.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Counter resets are the reader's problem, not this file's — tailscaled restarting returns
|
||||
// the counters to zero, and a sampler that tried to compensate would have to guess when.
|
||||
// Storing raw values leaves the evidence intact: a sample lower than the one before it is a
|
||||
// restart, and it is unambiguous.
|
||||
//
|
||||
// Trimmed to VV_MESH_KEEP_DAYS on every run, so the file cannot grow without bound. At one
|
||||
// sample per peer per minute that is ~1,440 lines/peer/day.
|
||||
//
|
||||
// Silent no-op when tailscale is absent or returns nothing parseable. This runs every minute
|
||||
// from a cache writer; a mesh sampler is not worth a log line per minute when it has nothing
|
||||
// to add.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php mesh_traffic_sample.php append one sample per mesh peer
|
||||
// php mesh_traffic_sample.php --show print what it would record, write nothing
|
||||
//
|
||||
// CONFIGURATION
|
||||
// master.conf
|
||||
// HOST1, HOST2, … the hostname of each node in the mesh. Every HOST<n> with a non-empty
|
||||
// value becomes a name matched against tailscale's peer list; a peer that
|
||||
// is not one of them is skipped, so a device that merely shares the
|
||||
// tailnet is never recorded as mesh traffic. No HOST<n> set at all means
|
||||
// there is no mesh to sample and the run exits without writing.
|
||||
//
|
||||
// Nothing is read from host*.conf. This samples the local tailscale daemon, so it needs to
|
||||
// know which peers count and nothing about how to reach them.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// tailscale status --json the counters
|
||||
// include/config.php DATA_DIR, vv_conf_vars() for the HOST* list
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
define('VV_MESH_DB', DATA_DIR . '/db/mesh_traffic.db');
|
||||
define('VV_MESH_KEEP_DAYS', 40); // a 30-day window needs a sample older than 30 days
|
||||
|
||||
$show = in_array('--show', $argv ?? [], true);
|
||||
|
||||
$raw = shell_exec('tailscale status --json 2>/dev/null');
|
||||
$js = json_decode((string)$raw, true);
|
||||
if (!is_array($js) || empty($js['Peer'])) exit(0);
|
||||
|
||||
// The hostnames this mesh is made of. Anything else on the tailnet is somebody's laptop.
|
||||
$vars = vv_conf_vars();
|
||||
$mesh = [];
|
||||
foreach ($vars as $k => $v) {
|
||||
if (preg_match('/^HOST\d+$/', $k) && trim((string)$v) !== '') $mesh[strtolower(trim($v))] = true;
|
||||
}
|
||||
if (!$mesh) exit(0);
|
||||
|
||||
$now = time();
|
||||
$lines = [];
|
||||
foreach ($js['Peer'] as $peer) {
|
||||
$host = strtolower(trim((string)($peer['HostName'] ?? '')));
|
||||
if ($host === '') continue;
|
||||
|
||||
// Same unambiguous-prefix rule the rest of the partnership layer uses: the tailnet name and
|
||||
// the conf hostname are different strings and nothing keeps them in step.
|
||||
$match = null;
|
||||
if (isset($mesh[$host])) {
|
||||
$match = $host;
|
||||
} else {
|
||||
$cand = [];
|
||||
foreach (array_keys($mesh) as $m) {
|
||||
if (str_starts_with($host, $m) || str_starts_with($m, $host)) $cand[] = $m;
|
||||
}
|
||||
if (count($cand) === 1) $match = $cand[0];
|
||||
}
|
||||
if ($match === null) continue;
|
||||
|
||||
$tx = (int)($peer['TxBytes'] ?? 0);
|
||||
$rx = (int)($peer['RxBytes'] ?? 0);
|
||||
if ($tx === 0 && $rx === 0) continue;
|
||||
$lines[] = $now . '|' . $match . '|' . $tx . '|' . $rx;
|
||||
}
|
||||
if (!$lines) exit(0);
|
||||
|
||||
if ($show) { echo implode("\n", $lines) . "\n"; exit(0); }
|
||||
|
||||
@mkdir(dirname(VV_MESH_DB), 0755, true);
|
||||
@file_put_contents(VV_MESH_DB, implode("\n", $lines) . "\n", FILE_APPEND | LOCK_EX);
|
||||
|
||||
// Trim in place. Read-filter-rewrite rather than append-only truncation, because the cut is by
|
||||
// age and the file is not ordered by peer.
|
||||
$cutoff = $now - (VV_MESH_KEEP_DAYS * 86400);
|
||||
$all = @file(VV_MESH_DB, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [];
|
||||
if (count($all) > 200) {
|
||||
$keep = [];
|
||||
foreach ($all as $l) {
|
||||
$ts = (int)strtok($l, '|');
|
||||
if ($ts >= $cutoff) $keep[] = $l;
|
||||
}
|
||||
if (count($keep) !== count($all)) {
|
||||
$tmp = VV_MESH_DB . '.tmp';
|
||||
if (@file_put_contents($tmp, implode("\n", $keep) . "\n") !== false) @rename($tmp, VV_MESH_DB);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Reproduce the owner's docker folder layout on the mirror, for the containers the owner
|
||||
// actually deployed there. Sonarr lands in "Arrs Stack", NginxProxyManager and Lldap in
|
||||
// "Networking", the databases in "Databases" — the same shelves they sit on at home.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Two modes, one file, piped over SSH:
|
||||
//
|
||||
// --export --containers=a,b,c on the OWNER. Reads folder.view3's docker.json, intersects
|
||||
// each folder with the deployed list, prints a JSON plan.
|
||||
// --import on the MIRROR. Reads that plan on stdin and upserts each
|
||||
// folder by name into BOTH the mirror's folder.view3 docker.json
|
||||
// and Varaverk's own docker_folders.json.
|
||||
//
|
||||
// Onboard Step 12 runs the pair. Nothing is assumed about the mirror's layout: folders it
|
||||
// already has are matched by name and extended, never duplicated or replaced.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The owner's layout is the source, and only the deployed subset travels.
|
||||
// "Arrs Stack" holds ten containers on the owner and five of them were deployed, so the
|
||||
// mirror's "Arrs Stack" gets those five. Sending the whole list would name containers the
|
||||
// mirror does not have, and folder.view3 renders those as empty tiles.
|
||||
//
|
||||
// One folder per owner folder — not one folder for everything.
|
||||
// Step 12 used to put all thirteen containers into "<Owner>-Fallback". That says whose
|
||||
// they are and nothing about what they do, and it is the wrong shape for a stack that runs
|
||||
// continuously rather than only during a failover.
|
||||
//
|
||||
// The fallback folder is for what is genuinely fallback-only.
|
||||
// Anything in PARTNERSHIP_FALLBACK_ONLY, plus anything deployed that the owner does not
|
||||
// file anywhere, goes to "<OwnerShort>-Fallback". A container that runs on the mirror all
|
||||
// the time belongs with its peers; a container that exists only to cover the owner going
|
||||
// dark belongs in a folder named after the owner.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Upsert by name, never append — folder.view3 keys by random id, so a blind write produces a
|
||||
// second folder with the same name and half the contents.
|
||||
//
|
||||
// Atomic write: .vv.tmp then rename(), so folder.view3 never reads a truncated file.
|
||||
//
|
||||
// Both stores or the layout is only half real. The Monitor container card reads Varaverk's
|
||||
// docker_folders.json, never folder.view3's, and include/docker.php seeds it exactly once —
|
||||
// when the file is absent. A mirror onboarded before this step existed had already been
|
||||
// seeded, so writing only folder.view3 left the card showing the old single lump forever.
|
||||
//
|
||||
// Absent plugin is no longer a skip on import. folder.view3 is optional; Varaverk's own
|
||||
// store is not, and it is the one the card reads. Export still needs the plugin and says so.
|
||||
//
|
||||
// Import trusts nothing about shape: every folder needs a non-empty name and an array of
|
||||
// container names, and anything else in the payload is ignored rather than merged.
|
||||
//
|
||||
// Icons are carried across as URLs. They are the owner's own icon values, already resolvable
|
||||
// from any host, and an icon that fails to load is cosmetic.
|
||||
//
|
||||
// REQUEST
|
||||
// mirror_folders.php --export --containers=Sonarr,Radarr[,…] [--fallback-only=a,b] [--owner=HOST1]
|
||||
// mirror_folders.php --import [--dry-run] (plan on stdin)
|
||||
//
|
||||
// RESPONSE
|
||||
// export: JSON plan on stdout — {"folders":[{"name":…,"icon":…,"containers":[…]}, …],
|
||||
// "unfiled":[…]}
|
||||
// import: one line per folder written, then a count
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// mirror_folders.php --export
|
||||
// Print this host's folder layout as JSON, for the owner to compute a plan from.
|
||||
//
|
||||
// mirror_folders.php --import
|
||||
// Read a layout on stdin and apply it to this host's folder.view3.
|
||||
//
|
||||
// mirror_folders.php --containers=<csv>
|
||||
// Restrict the operation to these container names.
|
||||
//
|
||||
// mirror_folders.php --fallback-only=<csv>
|
||||
// Place only these into the partner's fallback folder, leaving the rest where they are.
|
||||
//
|
||||
// --dry-run
|
||||
// Compute and report the plan without writing folder.view3. Combines with any of the above.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// /boot/config/plugins/folder.view3/docker.json the third-party layout, optional on import
|
||||
// SCRIPTS_DIR/docker_folders.json Varaverk's own layout — what the card reads
|
||||
// Tools/fallback_folder.php still owns the "<Owner>-Fallback" folder
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
$pluginDir = dirname(__DIR__);
|
||||
require_once $pluginDir . '/include/config.php';
|
||||
|
||||
define('FV3_JSON', '/boot/config/plugins/folder.view3/docker.json');
|
||||
// Varaverk's own store. Written as well as folder.view3's, because the Monitor container card
|
||||
// reads THIS file and nothing else — include/docker.php imports folder.view3 exactly once, when
|
||||
// this file does not yet exist. On a mirror onboarded before the layout step existed, that
|
||||
// bootstrap had already happened, so the card went on showing one lump "<Owner>-Fallback"
|
||||
// holding the entire auth and arr stacks while folder.view3 next to it showed the real shelves.
|
||||
define('VV_STORE', SCRIPTS_DIR . '/docker_folders.json');
|
||||
|
||||
$opts = getopt('', ['export', 'import', 'containers:', 'fallback-only:', 'dry-run']);
|
||||
|
||||
$listArg = function (string $k) use ($opts): array {
|
||||
if (!isset($opts[$k])) return [];
|
||||
return array_values(array_filter(array_map('trim', explode(',', (string)$opts[$k])), 'strlen'));
|
||||
};
|
||||
|
||||
// ── Load ─────────────────────────────────────────────────────────────────────────────────────
|
||||
// folder.view3 is optional on the mirror. Export needs it and has nothing to say without it;
|
||||
// import does not — Varaverk's own store is the one the card reads, and it is always writable.
|
||||
$haveFv3 = file_exists(FV3_JSON);
|
||||
if (!$haveFv3 && isset($opts['export'])) {
|
||||
echo json_encode(['folders' => [], 'unfiled' => $listArg('containers')]) . "\n";
|
||||
exit(0);
|
||||
}
|
||||
$fv3 = [];
|
||||
if ($haveFv3) {
|
||||
$fv3 = json_decode((string)@file_get_contents(FV3_JSON), true);
|
||||
if (!is_array($fv3)) { fwrite(STDERR, "folder.view3 docker.json is unreadable\n"); exit(1); }
|
||||
}
|
||||
// ── Export ───────────────────────────────────────────────────────────────────────────────────
|
||||
if (isset($opts['export'])) {
|
||||
$deployed = $listArg('containers');
|
||||
$fallbackOnly = array_map('strtolower', $listArg('fallback-only'));
|
||||
if (!$deployed) { fwrite(STDERR, "--containers is required for --export\n"); exit(2); }
|
||||
|
||||
// Case-insensitive membership, because container names come from XML filenames on one side
|
||||
// and docker on the other, and those have disagreed on capitalisation before (my-prowlarr).
|
||||
$remaining = [];
|
||||
foreach ($deployed as $c) $remaining[strtolower($c)] = $c;
|
||||
|
||||
$plan = [];
|
||||
foreach ($fv3 as $f) {
|
||||
if (!is_array($f)) continue;
|
||||
$name = trim((string)($f['name'] ?? ''));
|
||||
if ($name === '') continue;
|
||||
// The owner's own fallback folders describe the owner's coverage of someone else. They
|
||||
// are not part of the mirror's layout and copying them would be nonsense on that host.
|
||||
if (preg_match('/-Fallback$/i', $name)) continue;
|
||||
|
||||
$members = [];
|
||||
foreach ((array)($f['containers'] ?? []) as $c) {
|
||||
$lc = strtolower(trim((string)$c));
|
||||
if ($lc === '' || !isset($remaining[$lc])) continue;
|
||||
if (in_array($lc, $fallbackOnly, true)) continue; // claimed by the fallback folder
|
||||
$members[] = $remaining[$lc];
|
||||
unset($remaining[$lc]);
|
||||
}
|
||||
if ($members) $plan[] = ['name' => $name, 'icon' => (string)($f['icon'] ?? ''), 'containers' => $members];
|
||||
}
|
||||
|
||||
// Whatever the owner files nowhere, plus everything explicitly marked fallback-only.
|
||||
echo json_encode(['folders' => $plan, 'unfiled' => array_values($remaining)], JSON_UNESCAPED_SLASHES) . "\n";
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
// ── Import ───────────────────────────────────────────────────────────────────────────────────
|
||||
if (!isset($opts['import'])) {
|
||||
fwrite(STDERR, "usage: mirror_folders.php --export --containers=… | --import\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
$raw = stream_get_contents(STDIN);
|
||||
$plan = json_decode((string)$raw, true);
|
||||
if (!is_array($plan) || !isset($plan['folders']) || !is_array($plan['folders'])) {
|
||||
fwrite(STDERR, "import: no usable plan on stdin\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Applied to both stores, so they cannot drift apart the way they already did once. Takes a
|
||||
// store, returns the store with the plan folded in plus what changed — no writing, no printing,
|
||||
// because the two callers report differently.
|
||||
$applyPlan = function (array $store) use ($plan): array {
|
||||
$written = 0;
|
||||
$lines = [];
|
||||
foreach ($plan['folders'] as $spec) {
|
||||
if (!is_array($spec)) continue;
|
||||
$name = trim((string)($spec['name'] ?? ''));
|
||||
$cs = array_values(array_filter(array_map('trim', (array)($spec['containers'] ?? [])), 'strlen'));
|
||||
if ($name === '' || !$cs) continue;
|
||||
|
||||
$targetId = null;
|
||||
foreach ($store as $id => $f) {
|
||||
if (is_array($f) && strcasecmp((string)($f['name'] ?? ''), $name) === 0) { $targetId = $id; break; }
|
||||
}
|
||||
$created = false;
|
||||
if ($targetId === null) {
|
||||
// folder.view3's own id shape: 20 chars of url-safe base64.
|
||||
$targetId = substr(str_replace(['+', '/', '='], '', base64_encode(random_bytes(15))), 0, 20);
|
||||
$store[$targetId] = ['name' => $name, 'icon' => '', 'settings' => ['', '', '1', '', '1', ''],
|
||||
'regex' => '', 'containers' => [], 'containerImages' => []];
|
||||
$created = true;
|
||||
}
|
||||
$store[$targetId]['name'] = $name;
|
||||
// Only fill an icon that is missing — a mirror that has styled its own folder keeps its choice.
|
||||
if (($spec['icon'] ?? '') !== '' && trim((string)($store[$targetId]['icon'] ?? '')) === '') {
|
||||
$store[$targetId]['icon'] = (string)$spec['icon'];
|
||||
}
|
||||
$have = (array)($store[$targetId]['containers'] ?? []);
|
||||
$lc = array_map('strtolower', array_map('strval', $have));
|
||||
$added = 0;
|
||||
foreach ($cs as $c) {
|
||||
if (in_array(strtolower($c), $lc, true)) continue;
|
||||
$have[] = $c; $lc[] = strtolower($c); $added++;
|
||||
}
|
||||
$store[$targetId]['containers'] = array_values($have);
|
||||
$lines[] = sprintf(" %-7s %-22s +%d (%s)", $created ? 'create' : 'update', $name, $added, implode(', ', $cs));
|
||||
$written++;
|
||||
}
|
||||
|
||||
// ── Take the filed containers back out of any "-Fallback" folder ─────────────────────────
|
||||
// A container that now sits in "Arrs Stack" must not also sit in "Gmer4Lfe-Fallback". The
|
||||
// fallback folder answers "what is this host covering for the owner", and an earlier Step 12
|
||||
// dumped every deployed container into it — so it claimed the whole auth and arr stacks,
|
||||
// which run here continuously and are not failover coverage at all.
|
||||
//
|
||||
// Only containers this plan just filed are removed. Anything the operator put in that folder
|
||||
// by hand, or that the onboard filed there deliberately as fallback-only, is left alone.
|
||||
$filed = [];
|
||||
foreach ($plan['folders'] as $spec) {
|
||||
foreach ((array)($spec['containers'] ?? []) as $c) $filed[strtolower(trim((string)$c))] = true;
|
||||
}
|
||||
$pruned = 0;
|
||||
foreach ($store as $id => $f) {
|
||||
if (!is_array($f) || !preg_match('/-Fallback$/i', (string)($f['name'] ?? ''))) continue;
|
||||
$keep = [];
|
||||
foreach ((array)($f['containers'] ?? []) as $c) {
|
||||
if (isset($filed[strtolower(trim((string)$c))])) { $pruned++; continue; }
|
||||
$keep[] = $c;
|
||||
}
|
||||
$store[$id]['containers'] = array_values($keep);
|
||||
}
|
||||
return [$store, $written, $pruned, $lines];
|
||||
};
|
||||
|
||||
$dryRun = isset($opts['dry-run']);
|
||||
|
||||
// ── folder.view3's file ──────────────────────────────────────────────────────────────────────
|
||||
$fvWritten = 0;
|
||||
if ($haveFv3) {
|
||||
[$fv3, $fvWritten, $fvPruned, $fvLines] = $applyPlan($fv3);
|
||||
foreach ($fvLines as $l) echo $l . "\n";
|
||||
if ($fvPruned) printf(" pruned %d container(s) from -Fallback folder(s) — they are filed properly now\n", $fvPruned);
|
||||
if (($fvWritten || $fvPruned) && !$dryRun) {
|
||||
$tmp = FV3_JSON . '.vv.tmp';
|
||||
if (file_put_contents($tmp, json_encode($fv3, JSON_UNESCAPED_SLASHES)) === false || !rename($tmp, FV3_JSON)) {
|
||||
fwrite(STDERR, "failed to write " . FV3_JSON . "\n"); exit(1);
|
||||
}
|
||||
printf(" %d folder(s) written to folder.view3 ✅\n", $fvWritten);
|
||||
}
|
||||
} else {
|
||||
echo " folder.view3 not installed — Varaverk's own layout only\n";
|
||||
}
|
||||
|
||||
// ── Varaverk's own store ─────────────────────────────────────────────────────────────────────
|
||||
// Seeded from folder.view3 when it does not exist yet, which is the same bootstrap
|
||||
// include/docker.php performs — done here too so the very first import lands on a real layout
|
||||
// rather than an empty file.
|
||||
$vvStore = [];
|
||||
if (file_exists(VV_STORE)) {
|
||||
$vvStore = json_decode((string)@file_get_contents(VV_STORE), true);
|
||||
if (!is_array($vvStore)) $vvStore = [];
|
||||
} elseif ($haveFv3) {
|
||||
$vvStore = $fv3;
|
||||
}
|
||||
[$vvStore, $vvWritten, $vvPruned, ] = $applyPlan($vvStore);
|
||||
if (($vvWritten || $vvPruned) && !$dryRun) {
|
||||
@mkdir(dirname(VV_STORE), 0755, true);
|
||||
$tmp = VV_STORE . '.vv.tmp';
|
||||
if (file_put_contents($tmp, json_encode($vvStore, JSON_UNESCAPED_SLASHES)) === false || !rename($tmp, VV_STORE)) {
|
||||
fwrite(STDERR, "failed to write " . VV_STORE . "\n"); exit(1);
|
||||
}
|
||||
}
|
||||
printf(" %d folder(s) %s Varaverk's layout%s\n", $vvWritten,
|
||||
$dryRun ? 'would be written to' : 'written to',
|
||||
$vvPruned ? sprintf(" (%d unfiled from -Fallback)", $vvPruned) : '');
|
||||
|
||||
if ($dryRun) echo " DRY RUN — nothing written\n";
|
||||
if (!$fvWritten && !$vvWritten) echo " nothing to write\n";
|
||||
@@ -0,0 +1,85 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Accept one mesh message from a partner and store it. Invoked over SSH by the sending node's
|
||||
// vv_nc_deliver(), with the record on stdin.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Exit 0 means stored. The sender treats anything else as undelivered and spools for retry, so
|
||||
// a non-zero exit here is a message that will arrive later rather than one that is lost.
|
||||
//
|
||||
// Two jobs in one file because they are two halves of the same contract: --channel takes what
|
||||
// a partner is delivering now, --flush pushes what this host failed to deliver earlier. A node
|
||||
// is both a receiver and a sender, and splitting them would mean two files that must agree on
|
||||
// the same spool layout.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The exit code is the receipt, and it is the only one.
|
||||
// There is no acknowledgement message and no reply body. The sender is an SSH command that
|
||||
// already has an exit status, so inventing a second channel to say the same thing would
|
||||
// give the two ways to disagree.
|
||||
//
|
||||
// Refusing is cheaper than being wrong.
|
||||
// An unparseable record, an unknown channel or a failed write all exit non-zero and leave
|
||||
// the message in the sender's spool. A retry costs one SSH round trip; a message accepted
|
||||
// and dropped is gone with nothing recording that it existed.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Reached only over SSH with a key this mesh installed, so the caller already has root. This
|
||||
// file therefore validates shape, not authority — there is no privilege here to protect that
|
||||
// the transport has not already granted.
|
||||
//
|
||||
// The channel is resolved against this host's own membership. A name that is not a channel
|
||||
// this machine belongs to is refused, so the argument cannot address a path outside the store.
|
||||
//
|
||||
// Storage is append-and-trim through vv_nc_append(), which is idempotent on message id — a
|
||||
// retry of something that already landed is a no-op rather than a duplicate.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// Not invoked by hand — the sending node's vv_nc_deliver() runs it over SSH.
|
||||
//
|
||||
// php node_chat_receive.php --channel=<id> store one JSON record, read from stdin
|
||||
// php node_chat_receive.php --flush retry this host's own undelivered spool
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/node_chat.php vv_nc_append(), vv_nc_channel(), vv_nc_flush_spool()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once dirname(__DIR__) . '/include/node_chat.php';
|
||||
|
||||
$args = $argv ?? [];
|
||||
|
||||
if (in_array('--flush', $args, true)) {
|
||||
$n = vv_nc_flush_spool();
|
||||
if ($n) echo "delivered $n queued message(s)\n";
|
||||
exit(0);
|
||||
}
|
||||
|
||||
$channel = '';
|
||||
foreach ($args as $a) {
|
||||
if (str_starts_with($a, '--channel=')) $channel = substr($a, 10);
|
||||
}
|
||||
if ($channel === '' || !vv_nc_channel($channel)) {
|
||||
fwrite(STDERR, "unknown channel\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
$raw = stream_get_contents(STDIN);
|
||||
$msg = json_decode((string)$raw, true);
|
||||
if (!is_array($msg) || empty($msg['id']) || !isset($msg['text'])) {
|
||||
fwrite(STDERR, "malformed message\n");
|
||||
exit(2);
|
||||
}
|
||||
|
||||
// Rebuilt rather than stored as sent: a record is only ever the fields this store understands,
|
||||
// so a sender running newer code cannot write keys this one will later hand to a page.
|
||||
$clean = [
|
||||
'id' => preg_replace('/[^a-f0-9]/i', '', (string)$msg['id']),
|
||||
'ts' => (int)($msg['ts'] ?? time()),
|
||||
'from' => preg_replace('/[^a-z0-9]/i', '', strtolower((string)($msg['from'] ?? ''))),
|
||||
'kind' => in_array($msg['kind'] ?? 'msg', ['msg', 'question', 'notice'], true) ? $msg['kind'] : 'msg',
|
||||
'text' => mb_substr((string)$msg['text'], 0, 4000),
|
||||
'style' => vv_nc_clean_style(is_array($msg['style'] ?? null) ? $msg['style'] : []),
|
||||
];
|
||||
if ($clean['id'] === '' || $clean['from'] === '') { fwrite(STDERR, "malformed message\n"); exit(2); }
|
||||
|
||||
exit(vv_nc_append($channel, $clean) ? 0 : 1);
|
||||
@@ -0,0 +1,234 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Aggregates Nginx Proxy Manager's per-host access logs into a small store the Proxies tab can
|
||||
// read: how many requests each host has served, how many bytes went out, how the responses
|
||||
// broke down by status, and when it was last hit.
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// NPM writes one access log per proxy host and nothing that counts them. The logs on this
|
||||
// installation are 475 MB across 41 files — one of them 330 MB on its own — so the question
|
||||
// "how many times has this site been hit" cannot be answered inside a page load. This runs on a
|
||||
// schedule and leaves behind a few kilobytes of JSON.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Incremental. Each pass records the byte offset it reached in every log and starts there next
|
||||
// time, so the 475 MB is read once and each later pass reads only what has arrived since.
|
||||
//
|
||||
// Rotation is detected by the file being smaller than the offset already recorded. The counters
|
||||
// are cumulative and are never reset by it — but the lines that rotated out between two passes
|
||||
// are not counted, so a total is "requests seen since tracking began", not a claim about the
|
||||
// whole history of the host. Running daily keeps that gap to whatever NPM rotates in a day.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Read forward, never re-read.
|
||||
// fseek to the stored offset and read to the end. Re-parsing a 330 MB log every pass to
|
||||
// recompute a number that only grows is the kind of job that quietly becomes the reason a
|
||||
// nightly run takes an hour.
|
||||
//
|
||||
// A partial last line is not counted.
|
||||
// nginx is appending while this reads. The offset advances only to the end of the last
|
||||
// complete line, so the remainder is picked up whole on the next pass rather than parsed
|
||||
// as a truncated record and then parsed again.
|
||||
//
|
||||
// Totals only. No per-client or per-path breakdown is kept — that is an analytics product, and
|
||||
// this exists to answer "is anything using this host, and is it erroring".
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Non-fatal, always: a missing log directory, an unreadable file or an absent NPM exits 0.
|
||||
// One pass at a time, flock non-blocking.
|
||||
// The store is written tmp + rename and verified before it replaces the previous one.
|
||||
// A bounded amount of work per pass — VV_NPM_MAX_BYTES per file — so a log that grew enormously
|
||||
// between passes cannot make this run unboundedly long.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// npm_access_stats.php one pass
|
||||
// npm_access_stats.php --dry-run parse and report, write nothing
|
||||
// npm_access_stats.php --status print the store
|
||||
// npm_access_stats.php --reset forget offsets and totals, start again from the current logs
|
||||
//
|
||||
// CONFIGURATION
|
||||
// NPM_LOG_DIR where NPM's per-host logs live; derived from the container mount when unset
|
||||
// DB_DIR npm_access.json is written here
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
$dryRun = in_array('--dry-run', $argv, true);
|
||||
$status = in_array('--status', $argv, true);
|
||||
$reset = in_array('--reset', $argv, true);
|
||||
|
||||
// 512 MB per file per pass. The first pass over a 330 MB log is the only one that should ever come
|
||||
// near it; the cap exists so an unattended run cannot be surprised by a log that exploded.
|
||||
const VV_NPM_MAX_BYTES = 536870912;
|
||||
|
||||
function vv_npm_stats_path(): string {
|
||||
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/npm_access.json';
|
||||
}
|
||||
|
||||
// The host path to NPM's log directory. Taken from the container's own mount table rather than
|
||||
// hardcoded, because that mapping is the thing most likely to differ between installations.
|
||||
function vv_npm_log_dir(): string {
|
||||
$conf = trim(vv_conf_vars()['NPM_LOG_DIR'] ?? '');
|
||||
if ($conf !== '' && is_dir($conf)) return rtrim($conf, '/');
|
||||
$name = trim(vv_conf_vars()[strtoupper(vv_detect_host()) . '_NPM_CONTAINER'] ?? 'NginxProxyManager');
|
||||
// {{println}}, not a \n escape: the format string is passed through to docker as written, and
|
||||
// a backslash-n in a single-quoted PHP string arrives as two literal characters — which is how
|
||||
// this silently found no mounts and reported the log directory missing.
|
||||
$out = shell_exec('docker inspect ' . escapeshellarg($name)
|
||||
. ' --format ' . escapeshellarg('{{range .Mounts}}{{println .Source ":" .Destination}}{{end}}')
|
||||
. ' 2>/dev/null');
|
||||
foreach (explode("\n", trim((string) $out)) as $line) {
|
||||
// println space-separates its arguments, so the mapping arrives as "src : dst".
|
||||
$line = str_replace(' : ', ':', trim($line));
|
||||
[$src, $dst] = array_pad(explode(':', $line, 2), 2, '');
|
||||
if ($dst === '/config' && $src !== '' && is_dir("$src/log")) return "$src/log";
|
||||
if ($dst === '/data' && $src !== '' && is_dir("$src/logs")) return "$src/logs";
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function vv_npm_stats_read(): array {
|
||||
$p = vv_npm_stats_path();
|
||||
if (!is_file($p)) return ['hosts' => []];
|
||||
$j = json_decode((string) @file_get_contents($p), true);
|
||||
if (!is_array($j) || !isset($j['hosts']) || !is_array($j['hosts'])) return [];
|
||||
return $j;
|
||||
}
|
||||
|
||||
function vv_npm_stats_write(array $data): bool {
|
||||
$p = vv_npm_stats_path();
|
||||
if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false;
|
||||
$data['updated'] = time();
|
||||
$json = json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
|
||||
if ($json === false) return false;
|
||||
$tmp = $p . '.vv.tmp';
|
||||
if (@file_put_contents($tmp, $json) === false) return false;
|
||||
if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
|
||||
if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
// ── Status ────────────────────────────────────────────────────────────────────
|
||||
if ($status) {
|
||||
$s = vv_npm_stats_read();
|
||||
if (!$s) { echo "npm_access.json is unreadable\n"; exit(0); }
|
||||
printf("%-6s %12s %12s %8s %8s %8s %s\n", 'host', 'requests', 'sent', '2xx', '4xx', '5xx', 'last hit');
|
||||
foreach ($s['hosts'] ?? [] as $id => $h) {
|
||||
printf("%-6s %12s %12s %8s %8s %8s %s\n", $id,
|
||||
number_format($h['requests'] ?? 0), vv_npm_bytes($h['bytes'] ?? 0),
|
||||
number_format($h['s2xx'] ?? 0), number_format($h['s4xx'] ?? 0),
|
||||
number_format($h['s5xx'] ?? 0),
|
||||
!empty($h['last_seen']) ? date('Y-m-d H:i', $h['last_seen']) : '-');
|
||||
}
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Decimal, because this is a quantity of traffic and every network tool that will be compared
|
||||
// against it is decimal too. Memory is the binary one.
|
||||
function vv_npm_bytes(float $b): string {
|
||||
$u = ['B', 'kB', 'MB', 'GB', 'TB'];
|
||||
$i = 0;
|
||||
while ($b >= 1000 && $i < count($u) - 1) { $b /= 1000; $i++; }
|
||||
return round($b, $b < 10 && $i ? 1 : 0) . ' ' . $u[$i];
|
||||
}
|
||||
|
||||
$lock = @fopen(sys_get_temp_dir() . '/vv_npm_access.lock', 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
||||
|
||||
try {
|
||||
$dir = vv_npm_log_dir();
|
||||
if ($dir === '') { echo "NPM log directory not found — set NPM_LOG_DIR\n"; exit(0); }
|
||||
|
||||
$store = $reset ? ['hosts' => []] : vv_npm_stats_read();
|
||||
if (!$store) { echo "npm_access.json is malformed — refusing to overwrite it\n"; exit(1); }
|
||||
$hosts = $store['hosts'] ?? [];
|
||||
|
||||
$files = glob("$dir/proxy-host-*_access.log") ?: [];
|
||||
if (!$files) { echo "no per-host access logs in $dir\n"; exit(0); }
|
||||
|
||||
$t0 = microtime(true);
|
||||
$readTotal = 0; $newLines = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($files as $f) {
|
||||
if (!preg_match('/proxy-host-(\d+)_access\.log$/', $f, $m)) continue;
|
||||
$id = (string) (int) $m[1];
|
||||
$size = @filesize($f);
|
||||
if ($size === false) continue;
|
||||
|
||||
$h = $hosts[$id] ?? ['requests' => 0, 'bytes' => 0, 's2xx' => 0, 's3xx' => 0,
|
||||
's4xx' => 0, 's5xx' => 0, 'offset' => 0, 'last_seen' => null,
|
||||
'rotations' => 0, 'since' => time()];
|
||||
$off = (int) ($h['offset'] ?? 0);
|
||||
|
||||
// Smaller than where we stopped means the file was rotated out from under us. Counters are
|
||||
// cumulative and stay; only the offset resets, and the rotation is counted so the store can
|
||||
// say the totals have a gap in them.
|
||||
if ($size < $off) { $off = 0; $h['rotations'] = ($h['rotations'] ?? 0) + 1; }
|
||||
if ($size === $off) { $hosts[$id] = $h; continue; }
|
||||
|
||||
$fp = @fopen($f, 'rb');
|
||||
if (!$fp) { $hosts[$id] = $h; continue; }
|
||||
@fseek($fp, $off);
|
||||
|
||||
$budget = VV_NPM_MAX_BYTES;
|
||||
$read = 0;
|
||||
$lastCompleteOffset = $off;
|
||||
|
||||
while (!feof($fp) && $budget > 0) {
|
||||
$line = fgets($fp, 8192);
|
||||
if ($line === false) break;
|
||||
$len = strlen($line);
|
||||
$budget -= $len;
|
||||
$read += $len;
|
||||
// No trailing newline means nginx is mid-write. Stop and leave the offset before it.
|
||||
if (substr($line, -1) !== "\n") break;
|
||||
$lastCompleteOffset += $len;
|
||||
|
||||
// Varaverk's own uptime probe, dropped before it is counted. It hits every host once a
|
||||
// minute — fifty thousand requests a day — so counting it would make this monitor the
|
||||
// overwhelming majority of the traffic it reports, and a host nobody visits would look
|
||||
// as busy as one that is genuinely used. The string is VV_UPTIME_UA in uptime_probe.php
|
||||
// and the two must stay in step.
|
||||
if (strpos($line, 'Varaverk-Uptime/') !== false) { $skipped++; continue; }
|
||||
|
||||
// [09/Aug/2026:05:52:08 +0000] - 200 200 - GET https host "/" [Client 1.2.3.4] [Length 567] ...
|
||||
if (!preg_match('/^\[([^\]]+)\]\s+\S+\s+(\d{3})/', $line, $lm)) continue;
|
||||
$code = (int) $lm[2];
|
||||
$h['requests']++;
|
||||
$newLines++;
|
||||
if ($code >= 500) $h['s5xx']++;
|
||||
elseif ($code >= 400) $h['s4xx']++;
|
||||
elseif ($code >= 300) $h['s3xx']++;
|
||||
elseif ($code >= 200) $h['s2xx']++;
|
||||
if (preg_match('/\[Length (\d+)\]/', $line, $bm)) $h['bytes'] += (int) $bm[1];
|
||||
// The log stamp is nginx's own format; a line that will not parse is not worth a
|
||||
// guessed timestamp, so last_seen simply does not move for it.
|
||||
$ts = strtotime(str_replace('/', ' ', preg_replace('/^(\d+)\/(\w+)\/(\d+):/', '$1 $2 $3 ', $lm[1])));
|
||||
if ($ts !== false && ($h['last_seen'] === null || $ts > $h['last_seen'])) $h['last_seen'] = $ts;
|
||||
}
|
||||
fclose($fp);
|
||||
|
||||
$h['offset'] = $lastCompleteOffset;
|
||||
$hosts[$id] = $h;
|
||||
$readTotal += $read;
|
||||
}
|
||||
|
||||
ksort($hosts, SORT_NUMERIC);
|
||||
$store['hosts'] = $hosts;
|
||||
$store['last_pass'] = time();
|
||||
|
||||
printf("%d logs, read %s this pass, %s new requests, %s own probes ignored, %.1fs\n",
|
||||
count($files), vv_npm_bytes($readTotal), number_format($newLines),
|
||||
number_format($skipped), microtime(true) - $t0);
|
||||
|
||||
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
||||
if (!vv_npm_stats_write($store)) { echo 'could not write ' . vv_npm_stats_path() . "\n"; exit(1); }
|
||||
echo 'wrote ' . vv_npm_stats_path() . "\n";
|
||||
exit(0);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
Executable
+79
@@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================ NPM Access Stats ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Aggregates Nginx Proxy Manager's per-host access logs into DB_DIR/npm_access.json — requests,
|
||||
# bytes sent, status breakdown and last hit per proxy host. The Proxies tab reads it.
|
||||
#
|
||||
# NPM writes one access log per host and counts nothing. The logs here are 475 MB across 41 files,
|
||||
# so this cannot happen inside a page load; each pass reads only what arrived since the last one.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in npm_access_stats.php, next to the NPM client and conf helpers it uses.
|
||||
# Same split as api_cache_writer, ai_repair_sweep and cert_history.
|
||||
#
|
||||
# Totals are "since tracking began", not since the host existed: lines that rotate out between two
|
||||
# passes are not counted. Running daily keeps that to whatever NPM rotates in a day.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Incremental by byte offset, never a re-read.
|
||||
# 475 MB across 41 files cannot be parsed on a page load, and re-parsing what was already
|
||||
# counted would make each pass slower than the last. Each run records where it stopped and
|
||||
# resumes there, so cost tracks new traffic rather than total traffic.
|
||||
#
|
||||
# A rotated log is detected, not assumed.
|
||||
# A file smaller than the offset already recorded for it means NPM rotated it beneath us, so
|
||||
# that file's offset restarts at zero and a rotation is counted. The totals are cumulative and
|
||||
# are never reset by it — what rotated out between two passes is simply not counted, which is
|
||||
# why this runs often enough to keep that gap to whatever NPM rotates in a day.
|
||||
#
|
||||
# Varaverk's own probes are excluded.
|
||||
# uptime_probe.sh requests every proxied host every minute under Varaverk-Uptime/1.0. Counting
|
||||
# those would make the monitor the busiest client of everything it monitors.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Read-only against NPM. Access logs are read and never rotated, truncated or removed — that is
|
||||
# NPM's business, and a stats collector that deleted its own inputs would be unrecoverable.
|
||||
#
|
||||
# --dry-run parses and reports without writing the store or advancing any offset, so a dry run
|
||||
# leaves the next real pass with exactly the same work to do.
|
||||
#
|
||||
# --reset forgets offsets and totals deliberately. It starts from the current end of each log,
|
||||
# so it discards history rather than double-counting it.
|
||||
#
|
||||
# The store is replaced atomically: written to a temp file, re-read and parsed to prove it is
|
||||
# valid JSON, then renamed over the original. A pass that dies midway — or produces something
|
||||
# unparseable — leaves the previous document untouched rather than a truncated one.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# npm_access_stats.sh one pass
|
||||
# npm_access_stats.sh --dry-run parse and report, write nothing
|
||||
# npm_access_stats.sh --status print the store
|
||||
# npm_access_stats.sh --reset forget offsets and totals, start again from the current logs
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# NPM_LOG_DIR override the log directory; otherwise derived from the container's mounts
|
||||
# DB_DIR npm_access.json is written here
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/npm_access_stats.php" "$@"
|
||||
Executable → Regular
+28
-13
@@ -15,19 +15,6 @@
|
||||
# Accepts --host=HOST2 to refresh a single host (used by the UI refresh button).
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Cache-First, Never Live on Page Load
|
||||
# Remote arr APIs have non-trivial latency — calling them on every page view
|
||||
# would make the arrs page slow and fragile. Writing to /tmp/vv_cache/ on a
|
||||
# 2-hour schedule decouples page load time from network availability.
|
||||
#
|
||||
# Single-Host Refresh for UI
|
||||
# The UI refresh button passes --host=HOSTN to update one host without waiting
|
||||
# for the full 2-hour cycle. Keeps the cache fresh when a user requests it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -44,6 +31,19 @@
|
||||
# back to live calls when a file is missing or stale.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Cache-First, Never Live on Page Load
|
||||
# Remote arr APIs have non-trivial latency — calling them on every page view
|
||||
# would make the arrs page slow and fragile. Writing to /tmp/vv_cache/ on a
|
||||
# 2-hour schedule decouples page load time from network availability.
|
||||
#
|
||||
# Single-Host Refresh for UI
|
||||
# The UI refresh button passes --host=HOSTN to update one host without waiting
|
||||
# for the full 2-hour cycle. Keeps the cache fresh when a user requests it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -152,9 +152,11 @@ for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
"php -r \"
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/arrs.php';
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/unraid_api.php';
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/media.php';
|
||||
echo json_encode([
|
||||
'arrs' => vv_arrs_local_node(),
|
||||
'monitor' => vv_local_host_stats(),
|
||||
'media' => function_exists('vv_media_server_stats') ? vv_media_server_stats() : [],
|
||||
]);
|
||||
\"" 2>/dev/null)
|
||||
|
||||
@@ -188,6 +190,19 @@ for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
file_put_contents('$MONITOR_CACHE', json_encode(\$d['monitor']));
|
||||
" 2>/dev/null
|
||||
|
||||
# Save media server cache. Collected here rather than queried directly because a partner's
|
||||
# Emby URL is http://localhost:8096 — true on that host and meaningless from this one. The
|
||||
# same reason the arrs are collected this way: each host answers about itself, and its API
|
||||
# keys never leave it.
|
||||
#
|
||||
# Absent on a partner running an older build, which is why it is written only when present:
|
||||
# an empty file would read as "no media servers there" rather than "not collected yet".
|
||||
MEDIA_CACHE="$VV_CACHE_DIR/media_remote_${host_id}.json"
|
||||
echo "$RESULT" | php -r "
|
||||
\$d = json_decode(file_get_contents('php://stdin'), true);
|
||||
if (!empty(\$d['media'])) file_put_contents('$MEDIA_CACHE', json_encode(\$d['media']));
|
||||
" 2>/dev/null
|
||||
|
||||
CACHED_TYPES=$(echo "$RESULT" | php -r "
|
||||
\$d = json_decode(file_get_contents('php://stdin'), true);
|
||||
echo implode(', ', array_column(\$d['arrs']['arrs'] ?? [], 'type'));
|
||||
|
||||
@@ -281,12 +281,23 @@ echo "━━━ $ICON_GEAR Step 4: Update master.conf paths ━━━"
|
||||
NEW_MASTER="$DST/Configurations/master.conf"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ -f "$NEW_MASTER" ]]; then
|
||||
sed -i "s|^\(\s*TARGET_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST\"|" "$NEW_MASTER"
|
||||
sed -i "s|^\(\s*DATA_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST/data\"|" "$NEW_MASTER"
|
||||
sed -i "s|^\(\s*STATE_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST/State_Files\"|" "$NEW_MASTER"
|
||||
echo " TARGET_DIR → $DST ✅"
|
||||
echo " DATA_DIR → $DST/data ✅"
|
||||
echo " STATE_DIR → $DST/State_Files ✅"
|
||||
# Only rewrite a path that is a literal. These three are derived from SCRIPTS_DIR in
|
||||
# current confs, which load_config.sh exports from its own location — so after the move
|
||||
# they already point at the new install and hardcoding them here would turn a
|
||||
# self-correcting value back into one that has to be maintained. Older confs still carry
|
||||
# literals, so the substitution stays for them.
|
||||
_mig_path() { # $1=key $2=new value
|
||||
if grep -qE "^\s*$1\s*=\s*\"[^\"]*\\\$\{?(SCRIPTS_DIR|DATA_DIR)" "$NEW_MASTER"; then
|
||||
echo " $1 — already derived, left alone ✅"
|
||||
else
|
||||
sed -i "s|^\(\s*$1\s*=\s*\)\"[^\"]*\"|\1\"$2\"|" "$NEW_MASTER"
|
||||
echo " $1 → $2 ✅"
|
||||
fi
|
||||
}
|
||||
_mig_path TARGET_DIR "$DST"
|
||||
_mig_path DATA_DIR "$DST/data"
|
||||
_mig_path STATE_DIR "$DST/data/state"
|
||||
unset -f _mig_path
|
||||
else
|
||||
error "master.conf not found at $NEW_MASTER"
|
||||
exit 1
|
||||
@@ -326,7 +337,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
define('SCRIPTS_DIR', \$_c['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk');
|
||||
define('CONF_DIR', SCRIPTS_DIR . '/Configurations');
|
||||
define('DATA_DIR', SCRIPTS_DIR . '/data');
|
||||
define('STATE_DIR', SCRIPTS_DIR . '/State_Files');
|
||||
define('STATE_DIR', SCRIPTS_DIR . '/data/state');
|
||||
define('LOG_DIR', '/var/log/varaverk');
|
||||
define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db');
|
||||
require_once '/usr/local/emhttp/plugins/varaverk/include/confform.php';
|
||||
@@ -384,6 +395,24 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Repoint the WebGUI at the new location ────────────────────────────────────
|
||||
# The .plg rebuilds this symlink on every boot and now follows SCRIPTS_DIR, but a migration is
|
||||
# not a boot. Without this the WebGUI keeps serving the old copy until the next restart, so a
|
||||
# migration appears to work while every page, endpoint and pull-target is still the old tree.
|
||||
WEB_DIR="/usr/local/emhttp/plugins/varaverk"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would repoint $WEB_DIR → $DST/Plugin/unraid"
|
||||
elif [[ -d "$DST/Plugin/unraid" ]]; then
|
||||
ln -sfn "$DST/Plugin/unraid" "$WEB_DIR"
|
||||
if [[ "$(readlink -f "$WEB_DIR")" == "$(readlink -f "$DST/Plugin/unraid")" ]]; then
|
||||
echo "WebGUI repointed → $DST/Plugin/unraid ✅"
|
||||
else
|
||||
warn "Could not repoint $WEB_DIR — the WebGUI will keep serving the old tree until reboot"
|
||||
fi
|
||||
else
|
||||
warn "$DST/Plugin/unraid missing — WebGUI left pointing at the old tree"
|
||||
fi
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_DONE Migration complete ━━━━━"
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Writes pages/readme/ui-map.md — where every conf setting lives in the web UI, and the route
|
||||
// to reach it. Generated so the assistant can answer "how do I change X" with a path through
|
||||
// the pages instead of an instruction to open master.conf.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Reads the section registries and the conf files, resolves each setting to the page and card
|
||||
// that renders it, and writes the whole map in one pass. Nothing is merged with what is
|
||||
// already there — the output is derived entirely from the registries, so a stale entry cannot
|
||||
// survive a rebuild.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The assistant cannot see the UI any other way.
|
||||
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
|
||||
// useless if it were — a page is a pile of divs, not a description of itself — so the
|
||||
// assistant has never had any way to know the UI exists. It could name a conf key and
|
||||
// nothing more. pages/readme/*.md is the one directory the chunker classifies as
|
||||
// kind='ui', which is why the output lands there and not in docs/.
|
||||
//
|
||||
// Generated, because a hand-written map lies with confidence.
|
||||
// A second description of the pages starts being wrong the moment a card moves, and that
|
||||
// is worse than saying nothing, because the assistant will repeat it. Everything here is
|
||||
// derived from the same registries the pages themselves are built from:
|
||||
// VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script, VV_UI_SECTION_SURFACES
|
||||
// for the pages that show sections by subject, and the conf files for the settings and
|
||||
// their controls.
|
||||
//
|
||||
// An unreachable section is reported, never dropped.
|
||||
// A section no page renders is listed at the end rather than silently omitted. A setting
|
||||
// with no route through the UI is a real finding, and this map is the only thing that
|
||||
// would ever notice.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Writes exactly one file, pages/readme/ui-map.md, and nothing else. No conf is modified, no
|
||||
// page is touched, and the registries it reads are only read.
|
||||
//
|
||||
// --check reports what would change and writes nothing, so the map can be verified current in
|
||||
// a commit without regenerating it.
|
||||
//
|
||||
// Generated output only. Nothing hand-edited belongs in ui-map.md — an edit there is lost on
|
||||
// the next run, which is the correct behaviour for a derived file and the reason the header
|
||||
// says so.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// php Tools/ui_map_build.php write the map
|
||||
// php Tools/ui_map_build.php --check report what it would change, write nothing
|
||||
//
|
||||
// Hand-run. Re-run after adding a conf section, a script mapping or a settings surface.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/confform.php the section registries, the parser, and the inferred controls
|
||||
// include/scheduler.php vv_pretty_label() — the same script naming the Scheduler cards use,
|
||||
// so a route names the card the operator is actually looking for
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
require_once __DIR__ . '/../include/confform.php';
|
||||
require_once __DIR__ . '/../include/scheduler.php';
|
||||
|
||||
$check = in_array('--check', $argv, true);
|
||||
$outRel = 'Plugin/unraid/pages/readme/ui-map.md';
|
||||
$outAbs = dirname(__DIR__) . '/pages/readme/ui-map.md';
|
||||
|
||||
// ── Gather every section in every conf file this host can see ────────────────────────────────
|
||||
$sections = []; // "file\0subsection" => ['file','subsection','fields']
|
||||
foreach (vv_get_conf_files() as $file) {
|
||||
foreach (vv_conf_all_groups($file) as $g) {
|
||||
if (empty($g['fields'])) continue;
|
||||
$sections[$file . "\0" . $g['subsection']] = $g;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Route 1: the Scheduler, one script at a time ─────────────────────────────────────────────
|
||||
// A section may be reached through several scripts — a shared threshold belongs to whichever
|
||||
// scripts read it — so routes accumulate rather than overwrite.
|
||||
$routes = []; // section key => list of human routes
|
||||
foreach (VV_SCRIPT_CONF_SECTIONS as $script => $subs) {
|
||||
$label = vv_pretty_label(basename($script, '.sh'));
|
||||
foreach ((array) $subs as $sub) {
|
||||
foreach ($sections as $k => $g) {
|
||||
if (strcasecmp($g['subsection'], $sub) !== 0) continue;
|
||||
$routes[$k][] = "Scheduler tab → **{$label}** → Config → *{$g['subsection']}*";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Route 2: pages that show sections by subject ─────────────────────────────────────────────
|
||||
foreach (VV_UI_SECTION_SURFACES as $surface) {
|
||||
// "*" is the catch-all, matched the same way api/confform.php matches it. preg_quote would
|
||||
// turn it into \* and quietly match nothing, which is how the map went on reporting a third
|
||||
// of the conf as unreachable after the page that reaches it had shipped.
|
||||
// Same matching as api/confform.php, including the pipe-separated form — the map and the page
|
||||
// must agree about which sections a surface shows, or the route it prints is fiction.
|
||||
$all = ((string) $surface['match'] === '*');
|
||||
$re = $all ? '' : '/\b(' . implode('|', array_map(
|
||||
fn($w) => preg_quote(trim($w), '/'),
|
||||
array_filter(explode('|', (string) $surface['match']), fn($w) => trim($w) !== '')))
|
||||
. ')\b/i';
|
||||
foreach ($sections as $k => $g) {
|
||||
$name = (string) $g['subsection'];
|
||||
if (isset(VV_UI_SECTION_EXCLUDE[$name])) continue;
|
||||
if (!$all && !preg_match($re, $name)) continue;
|
||||
$routes[$k][] = $surface['route'] . " → *{$name}*";
|
||||
}
|
||||
}
|
||||
|
||||
// ── Route 3: pages with a purpose-built control for one named setting ────────────────────────
|
||||
// Read out of the page source rather than declared, so a control that is added or removed moves
|
||||
// the map with it. The pattern is the literal key in a change payload or a toggle call — the one
|
||||
// shape these pages have in common. Keys assembled at runtime are invisible here and correctly
|
||||
// fall through to the conf-only list rather than being guessed at.
|
||||
// Every key this host's confs actually define, so a declared route can be checked against
|
||||
// reality rather than trusted.
|
||||
$known = [];
|
||||
foreach ($sections as $g) foreach ($g['fields'] as $f) $known[$f['key']] = true;
|
||||
|
||||
// This machine's host slot, for substituting HOSTN in declared keys.
|
||||
$slot = 'HOST1';
|
||||
foreach (vv_get_conf_files() as $f) {
|
||||
if (preg_match('/^host(\d+)\.conf$/i', $f, $hm)) { $slot = 'HOST' . $hm[1]; break; }
|
||||
}
|
||||
|
||||
$keyRoutes = []; // KEY => list of routes
|
||||
$stale = [];
|
||||
foreach (VV_UI_PAGE_ROUTES as $page => $spec) {
|
||||
$route = is_array($spec) ? $spec['route'] : $spec;
|
||||
$src = @file_get_contents(dirname(__DIR__) . '/pages/' . $page);
|
||||
if ($src === false) { fwrite(STDERR, "note: $page not found, skipped\n"); continue; }
|
||||
if (preg_match_all('/(?:key|name)\s*:\s*\'([A-Z][A-Z0-9_]{3,})\'|\(this,\s*\'([A-Z][A-Z0-9_]{3,})\'\)/',
|
||||
$src, $m, PREG_SET_ORDER)) {
|
||||
foreach ($m as $hit) {
|
||||
$key = $hit[1] !== '' ? $hit[1] : ($hit[2] ?? '');
|
||||
if ($key !== '' && isset($known[$key])) $keyRoutes[$key][] = $route;
|
||||
}
|
||||
}
|
||||
foreach ((array) (is_array($spec) ? ($spec['also'] ?? []) : []) as $decl) {
|
||||
$key = str_replace('HOSTN', $slot, $decl);
|
||||
if (isset($known[$key])) { $keyRoutes[$key][] = $route; continue; }
|
||||
$stale[] = "$page declares $decl (→ $key) which no conf defines";
|
||||
}
|
||||
}
|
||||
|
||||
// ── How each control is described to someone who has to find it ──────────────────────────────
|
||||
const UI_CONTROL_WORDS = [
|
||||
'bool' => 'a switch',
|
||||
'int' => 'a number box',
|
||||
'enum' => 'a dropdown',
|
||||
'secret' => 'a masked box with a **Show** button',
|
||||
'lines' => 'a list, one entry per line',
|
||||
'path' => 'a text box',
|
||||
'text' => 'a text box',
|
||||
];
|
||||
|
||||
$md = "# Where every setting lives in the web UI\n\n";
|
||||
$md .= "Generated by `Tools/ui_map_build.php` — do not edit by hand.\n\n";
|
||||
$md .= "Every setting below can be changed in the browser. Nothing here needs a conf file opened\n"
|
||||
. "over SSH, and the routes are what to tell someone who asks where a setting is.\n\n";
|
||||
$md .= "Two surfaces show settings, and which one holds a given section depends on what the\n"
|
||||
. "section is about:\n\n";
|
||||
$md .= "- **The Scheduler tab** shows the settings belonging to one script. Pick the script, open\n"
|
||||
. " **Config**, and its sections appear there.\n";
|
||||
$md .= "- **The AI tab** shows the AI sections together under **Settings → Configuration**.\n\n";
|
||||
$md .= "Both write through the same guarded path: the change is validated, the conf is backed up,\n"
|
||||
. "the result is syntax-checked and read back, and a bad write is rolled back.\n\n";
|
||||
$md .= "A setting is edited by finding its row and changing the control described below. The Save\n"
|
||||
. "button sends only what was actually changed.\n\n";
|
||||
$md .= "**When someone asks where a setting is, answer with the route.** Every setting here has a\n"
|
||||
. "control in the browser, so the conf file it lands in is background rather than an\n"
|
||||
. "instruction — telling someone to edit the file by hand is the wrong answer when a switch\n"
|
||||
. "exists, and it is also the riskier one. The exception is the list at the very bottom:\n"
|
||||
. "those settings genuinely have no control, and saying so is the right answer.\n\n---\n";
|
||||
|
||||
// A section with no section-level route may still have per-key routes, if a page carries a
|
||||
// purpose-built control for some of its settings. That is a reachable section — just one whose
|
||||
// route is stated per row rather than once at the top.
|
||||
foreach ($sections as $k => $g) {
|
||||
if (!empty($routes[$k])) continue;
|
||||
foreach ($g['fields'] as $f) {
|
||||
if (!empty($keyRoutes[$f['key']])) { $routes[$k][] = '__perkey__'; break; }
|
||||
}
|
||||
}
|
||||
|
||||
$reachable = 0; $unreachable = [];
|
||||
ksort($sections);
|
||||
foreach ($sections as $k => $g) {
|
||||
if (empty($routes[$k])) { $unreachable[] = $g; continue; }
|
||||
$reachable++;
|
||||
|
||||
$md .= "\n## " . $g['subsection'] . "\n\n";
|
||||
|
||||
// The route leads. Naming the conf file first invited answers that told the operator to edit
|
||||
// host1.conf and mentioned the tab as an afterthought — which is the habit this file exists
|
||||
// to break. The file is still stated, because "where does this end up" is a fair question,
|
||||
// but it is stated last and as a fact rather than as an instruction.
|
||||
$seen = array_values(array_diff(array_unique($routes[$k]), ['__perkey__']));
|
||||
if (!$seen) {
|
||||
$md .= "No single page shows this section. Individual settings below carry their own route.\n\n";
|
||||
} else {
|
||||
$md .= count($seen) === 1 ? "Route: " . $seen[0] . "\n\n"
|
||||
: "Reachable from:\n\n" . implode("\n", array_map(fn($r) => "- $r", $seen)) . "\n\n";
|
||||
}
|
||||
|
||||
$md .= "Saved into `" . $g['file'] . "`, which does not need to be opened by hand.\n\n";
|
||||
$md .= "| Setting | Control | Where | What it does |\n|---|---|---|---|\n";
|
||||
foreach ($g['fields'] as $f) {
|
||||
$ctl = UI_CONTROL_WORDS[$f['widget'] ?? 'text'] ?? 'a text box';
|
||||
if (($f['widget'] ?? '') === 'enum' && !empty($f['choices'])) {
|
||||
$ctl .= ' (' . implode(', ', array_map(fn($c) => $c['value'], $f['choices'])) . ')';
|
||||
}
|
||||
if (!empty($f['unit'])) $ctl .= ', in ' . $f['unit'];
|
||||
if (isset($f['min'])) $ctl .= ', ' . $f['min'] . '–' . $f['max'];
|
||||
// The conf's own comment. Newlines and pipes would break the table row.
|
||||
$desc = trim(preg_replace('/\s+/', ' ', (string) ($f['desc'] ?? '')));
|
||||
$desc = str_replace('|', '\\|', $desc);
|
||||
if (mb_strlen($desc) > 400) $desc = mb_substr($desc, 0, 397) . '…';
|
||||
// A per-key route wins for that row: a purpose-built control is a better answer than
|
||||
// "somewhere in this section", and it is often on a different page entirely.
|
||||
$where = !empty($keyRoutes[$f['key']])
|
||||
? implode(', ', array_unique($keyRoutes[$f['key']]))
|
||||
: ($seen ? 'in this section' : '—');
|
||||
$md .= '| `' . $f['key'] . '` | ' . $ctl . ' | ' . $where . ' | '
|
||||
. ($desc !== '' ? $desc : '—') . " |\n";
|
||||
}
|
||||
}
|
||||
|
||||
// Two different kinds of "no route", and conflating them was unhelpful. One is a gap; the other
|
||||
// is a decision, and the decision has a reason worth repeating to whoever asks.
|
||||
$excluded = array_filter($unreachable, fn($g) => isset(VV_UI_SECTION_EXCLUDE[$g['subsection']]));
|
||||
$gaps = array_filter($unreachable, fn($g) => !isset(VV_UI_SECTION_EXCLUDE[$g['subsection']]));
|
||||
|
||||
if ($excluded) {
|
||||
$md .= "\n---\n\n## Settings deliberately kept out of the UI\n\n";
|
||||
$md .= "These have no control on purpose. They hold the machine's identity and the roots\n"
|
||||
. "everything else is derived from, and a text box beside a Save button is the wrong\n"
|
||||
. "shape for a value that decides whether the server recognises itself on next boot.\n\n"
|
||||
. "If one is asked about, give the reason and say it is edited in the conf file directly.\n"
|
||||
. "Do not describe a route — there is none, and that is the point.\n\n";
|
||||
foreach ($excluded as $g) {
|
||||
$keys = implode(', ', array_map(fn($f) => '`' . $f['key'] . '`', $g['fields']));
|
||||
$md .= '- **' . $g['subsection'] . '** (`' . $g['file'] . "`) — $keys \n"
|
||||
. ' ' . VV_UI_SECTION_EXCLUDE[$g['subsection']] . "\n";
|
||||
}
|
||||
}
|
||||
|
||||
if ($gaps) {
|
||||
$md .= "\n---\n\n## Settings with no route through the UI\n\n";
|
||||
$md .= "Not a decision, just not built yet: no page renders these, so they can only be changed\n"
|
||||
. "by editing the conf file. If one of these is asked about, say so plainly rather than\n"
|
||||
. "inventing a route.\n\n";
|
||||
foreach ($gaps as $g) {
|
||||
$keys = implode(', ', array_map(fn($f) => '`' . $f['key'] . '`', $g['fields']));
|
||||
$md .= '- **' . $g['subsection'] . '** (`' . $g['file'] . "`) — $keys\n";
|
||||
}
|
||||
}
|
||||
|
||||
$existing = is_readable($outAbs) ? file_get_contents($outAbs) : null;
|
||||
$same = $existing !== null && $existing === $md;
|
||||
|
||||
foreach ($stale as $s) fwrite(STDERR, "STALE ROUTE: $s\n");
|
||||
printf("%d sections reachable, %d with no UI route\n", $reachable, count($unreachable));
|
||||
printf("%d settings documented\n", array_sum(array_map(
|
||||
fn($k) => empty($routes[$k]) ? 0 : count($sections[$k]['fields']), array_keys($sections))));
|
||||
|
||||
if ($check) {
|
||||
echo $same ? "up to date\n" : "OUT OF DATE — re-run without --check\n";
|
||||
exit($same ? 0 : 1);
|
||||
}
|
||||
if ($same) { echo "no change\n"; exit(0); }
|
||||
|
||||
if (@file_put_contents($outAbs, $md) === false) {
|
||||
fwrite(STDERR, "could not write $outAbs\n");
|
||||
exit(1);
|
||||
}
|
||||
printf("wrote %s (%d bytes)\n", $outRel, strlen($md));
|
||||
@@ -0,0 +1,337 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Probes every hostname NPM serves, from outside the proxy, and records whether it answered.
|
||||
// Produces the uptime percentages and the recent-history strip on the Proxies tab, and the list
|
||||
// of anything that had a wobble for the Sunday report.
|
||||
//
|
||||
// WHY IT EXISTS
|
||||
// NPM's access log says what happened when somebody visited. It says nothing at all about a host
|
||||
// nobody visited, which is exactly the host most likely to be quietly broken. Five hosts here
|
||||
// have been returning errors on every request for months without anything noticing.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// One pass = one sample per domain, all issued in parallel through curl_multi so the wall time
|
||||
// is roughly the slowest domain rather than the sum of all of them. Intended to run every
|
||||
// minute; it is safe at any interval because everything is stored as counts and buckets rather
|
||||
// than assuming a cadence.
|
||||
//
|
||||
// up = the host answered with an HTTP status below 500.
|
||||
// down = nothing answered, the TLS handshake failed, or it answered 5xx.
|
||||
//
|
||||
// A 401 or a 302 to the Authelia portal is UP. The question this asks is "is the server there",
|
||||
// and an auth redirect is the strongest possible evidence that it is. Counting a protected site
|
||||
// as down would mark every guarded host on this installation permanently offline.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Probes carry a User-Agent that the access-log aggregator ignores.
|
||||
// Thirty-five domains once a minute is fifty thousand requests a day landing in the very
|
||||
// logs Tools/npm_access_stats.sh counts. Left unmarked, this monitor would become the
|
||||
// majority of the traffic it reports and bury whatever real use these hosts get.
|
||||
//
|
||||
// Bounded storage, decided up front.
|
||||
// Per domain: lifetime counters, the last 60 samples for the strip, 48 hourly buckets,
|
||||
// 30 daily ones and 12 monthly ones. That is a fixed size — a per-minute probe kept as raw
|
||||
// samples would be fifty thousand rows a day and the file would be the problem instead of
|
||||
// the answer. Each tier is what one view on the Proxies tab draws: 24h, 7d, 30d, 12 months.
|
||||
//
|
||||
// Bounded in domains too: a record nothing has probed for VV_DOMAIN_DROP_DAYS is retired,
|
||||
// so hosts removed or renamed in NPM do not accumulate for ever under the live ones.
|
||||
//
|
||||
// The domain list follows NPM, not a hand-kept list in conf.
|
||||
// A host added in the Proxies tab starts being probed without anyone remembering to add it
|
||||
// somewhere else. The list is cached so this does not call the NPM API every minute.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Non-fatal, always: no NPM, no credentials, no domains — exits 0.
|
||||
// One pass at a time, flock non-blocking, so a slow pass cannot overlap the next minute's.
|
||||
// Store written tmp + rename, verified before it replaces the previous file.
|
||||
// HEAD, not GET, and nothing is followed — this is a liveness check, not a crawler.
|
||||
//
|
||||
// RUNTIME MODES
|
||||
// uptime_probe.php one pass
|
||||
// uptime_probe.php --dry-run probe and report, write nothing
|
||||
// uptime_probe.php --status per-domain uptime table
|
||||
// uptime_probe.php --events recent state changes, newest first
|
||||
//
|
||||
// CONFIGURATION
|
||||
// UPTIME_PROBE_ENABLED master switch (default true)
|
||||
// UPTIME_PROBE_TIMEOUT seconds per domain (default 8)
|
||||
// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 300)
|
||||
// VV_UPTIME_UA the User-Agent, matched by npm_access_stats.php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
// Shared with npm_access_stats.php, which drops any log line containing it. Changing this in one
|
||||
// place and not the other turns the monitor's own traffic back into counted requests.
|
||||
const VV_UPTIME_UA = 'Varaverk-Uptime/1.0';
|
||||
const VV_SAMPLES_KEEP = 60; // one hour at a one-minute cadence
|
||||
const VV_HOURS_KEEP = 48;
|
||||
const VV_DAYS_KEEP = 30;
|
||||
// A year as twelve monthly buckets rather than 365 daily ones. The yearly view is a shape — which
|
||||
// months were bad — not a date lookup, and rolling the day buckets out to 365 would grow the file
|
||||
// twelvefold to answer the same question at a resolution nothing displays.
|
||||
const VV_MONTHS_KEEP = 12;
|
||||
// A domain removed or renamed in NPM stops being probed, and its record would otherwise sit here
|
||||
// for ever — the store only ever gained keys before this. 90 days rather than something tighter
|
||||
// because a host switched off for a season is a normal thing here and its history should survive
|
||||
// that; a domain nobody has served for a quarter is gone for good.
|
||||
const VV_DOMAIN_DROP_DAYS = 90;
|
||||
const VV_EVENTS_KEEP = 20;
|
||||
|
||||
$dryRun = in_array('--dry-run', $argv, true);
|
||||
$status = in_array('--status', $argv, true);
|
||||
$events = in_array('--events', $argv, true);
|
||||
$report = in_array('--report', $argv, true);
|
||||
|
||||
function vv_uptime_path(): string {
|
||||
return rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/uptime.json';
|
||||
}
|
||||
|
||||
function vv_uptime_read(): array {
|
||||
$p = vv_uptime_path();
|
||||
if (!is_file($p)) return ['domains' => []];
|
||||
$j = json_decode((string) @file_get_contents($p), true);
|
||||
if (!is_array($j) || !isset($j['domains']) || !is_array($j['domains'])) return [];
|
||||
return $j;
|
||||
}
|
||||
|
||||
function vv_uptime_write(array $d): bool {
|
||||
$p = vv_uptime_path();
|
||||
if (!is_dir(dirname($p)) && !@mkdir(dirname($p), 0755, true)) return false;
|
||||
$d['updated'] = time();
|
||||
$json = json_encode($d, JSON_UNESCAPED_SLASHES);
|
||||
if ($json === false) return false;
|
||||
$tmp = $p . '.vv.tmp';
|
||||
if (@file_put_contents($tmp, $json) === false) return false;
|
||||
if (json_decode((string) @file_get_contents($tmp), true) === null) { @unlink($tmp); return false; }
|
||||
if (!@rename($tmp, $p)) { @unlink($tmp); return false; }
|
||||
return true;
|
||||
}
|
||||
|
||||
// ── Report ────────────────────────────────────────────────────────────────────
|
||||
// Anything that was not perfect over the last seven days, for the Sunday report. Prints nothing
|
||||
// and exits 0 when every domain was clean — the orchestrator's job is to be quiet on a good week,
|
||||
// and a report that always says something is a report nobody reads.
|
||||
if ($report) {
|
||||
$s = vv_uptime_read();
|
||||
$doms = $s['domains'] ?? [];
|
||||
if (!$doms) { exit(0); }
|
||||
$down = $wobble = [];
|
||||
foreach ($doms as $d => $r) {
|
||||
$w = vv_uptime_window($r['days'] ?? [], 7);
|
||||
if (($r['state'] ?? '') === 'down') {
|
||||
$since = !empty($r['last_change']) ? ' since ' . date('D H:i', $r['last_change']) : '';
|
||||
$down[] = sprintf(' %-34s DOWN%s — %s', $d, $since, $r['last_detail'] ?? '');
|
||||
} elseif ($w !== null && $w < 100) {
|
||||
$wobble[] = sprintf(' %-34s %.2f%% over 7 days', $d, $w);
|
||||
}
|
||||
}
|
||||
if (!$down && !$wobble) exit(0);
|
||||
echo "Uptime — 7 day review\n";
|
||||
if ($down) { echo "\nCurrently down:\n"; foreach ($down as $l) echo "$l\n"; }
|
||||
if ($wobble) { echo "\nNot perfect this week:\n"; foreach ($wobble as $l) echo "$l\n"; }
|
||||
// Non-zero so the calling wrapper can notify on "there is something to say" without parsing.
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// ── Status / events ───────────────────────────────────────────────────────────
|
||||
if ($status || $events) {
|
||||
$s = vv_uptime_read();
|
||||
if (!$s) { echo "uptime.json is unreadable\n"; exit(0); }
|
||||
$doms = $s['domains'] ?? [];
|
||||
if ($events) {
|
||||
$all = [];
|
||||
foreach ($doms as $d => $r) foreach ($r['events'] ?? [] as $e) $all[] = $e + ['domain' => $d];
|
||||
usort($all, fn($a, $b) => $b['ts'] <=> $a['ts']);
|
||||
if (!$all) { echo "no state changes recorded\n"; exit(0); }
|
||||
foreach (array_slice($all, 0, 40) as $e)
|
||||
printf("%s %-34s %-5s %s\n", date('Y-m-d H:i', $e['ts']), $e['domain'],
|
||||
strtoupper($e['to']), $e['detail'] ?? '');
|
||||
exit(0);
|
||||
}
|
||||
printf("%-34s %-6s %8s %8s %8s %8s %s\n", 'domain', 'state', '24h', '7d', '30d', '1y', 'since');
|
||||
foreach ($doms as $d => $r) {
|
||||
$h24 = vv_uptime_window($r['hours'] ?? [], 24);
|
||||
$d7 = vv_uptime_window($r['days'] ?? [], 7);
|
||||
$d30 = vv_uptime_window($r['days'] ?? [], 30);
|
||||
$y1 = vv_uptime_window($r['months'] ?? [], 12);
|
||||
printf("%-34s %-6s %8s %8s %8s %8s %s\n", substr($d, 0, 34), $r['state'] ?? '-',
|
||||
$h24 === null ? '-' : $h24 . '%', $d7 === null ? '-' : $d7 . '%',
|
||||
$d30 === null ? '-' : $d30 . '%', $y1 === null ? '-' : $y1 . '%',
|
||||
!empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-');
|
||||
}
|
||||
printf("\n%d domains, last pass %s\n", count($doms),
|
||||
!empty($s['last_pass']) ? date('Y-m-d H:i:s', $s['last_pass']) : 'never');
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Shared by --status, --report and the API, and defined once in include/auth.php so the three
|
||||
// callers cannot drift apart on what "the last N" means. Buckets are keyed by time, so it is a key
|
||||
// sort rather than an assumption about how many samples a period should contain — a pass that did
|
||||
// not run leaves no bucket rather than a zero.
|
||||
function vv_uptime_window(array $buckets, int $n): ?float {
|
||||
return vv_auth_uptime_window($buckets, $n);
|
||||
}
|
||||
|
||||
// ── One pass ──────────────────────────────────────────────────────────────────
|
||||
$lock = @fopen(sys_get_temp_dir() . '/vv_uptime_probe.lock', 'c');
|
||||
if ($lock === false || !flock($lock, LOCK_EX | LOCK_NB)) { echo "another pass is running\n"; exit(0); }
|
||||
|
||||
try {
|
||||
$v = vv_conf_vars();
|
||||
if (strtolower(trim($v['UPTIME_PROBE_ENABLED'] ?? 'true')) === 'false') {
|
||||
echo "UPTIME_PROBE_ENABLED is false\n"; exit(0);
|
||||
}
|
||||
$timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8));
|
||||
$listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 300));
|
||||
|
||||
$store = vv_uptime_read();
|
||||
if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); }
|
||||
|
||||
// The domain list, refreshed occasionally rather than every minute. NPM's API is the source of
|
||||
// truth for what is being served; asking it sixty times an hour for a list that changes weekly
|
||||
// is the sort of thing that shows up later as unexplained load.
|
||||
$now = time();
|
||||
if (empty($store['list']) || ($now - ($store['list_at'] ?? 0)) > $listTtl) {
|
||||
$p = vv_npm_list_proxies();
|
||||
if ($p['ok'] ?? false) {
|
||||
$list = [];
|
||||
foreach ($p['proxies'] as $h) {
|
||||
// A disabled host is not expected to answer, so probing it would record a
|
||||
// permanent outage for something switched off on purpose.
|
||||
if (($h['enabled'] ?? true) === false) continue;
|
||||
foreach ($h['domain_names'] ?? [] as $d) {
|
||||
$d = strtolower(trim((string) $d));
|
||||
// A wildcard is not a hostname you can connect to.
|
||||
if ($d !== '' && !str_contains($d, '*')) $list[$d] = true;
|
||||
}
|
||||
}
|
||||
if ($list) { $store['list'] = array_keys($list); $store['list_at'] = $now; }
|
||||
}
|
||||
}
|
||||
$domains = $store['list'] ?? [];
|
||||
if (!$domains) { echo "no domains to probe\n"; exit(0); }
|
||||
|
||||
// ── Probe, in parallel ──
|
||||
$mh = curl_multi_init();
|
||||
$handles = [];
|
||||
foreach ($domains as $d) {
|
||||
$ch = curl_init('https://' . $d . '/');
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_NOBODY => true, // HEAD — liveness, not content
|
||||
CURLOPT_FOLLOWLOCATION => false, // a redirect to the auth portal is the answer
|
||||
CURLOPT_TIMEOUT => $timeout,
|
||||
CURLOPT_CONNECTTIMEOUT => min($timeout, 5),
|
||||
CURLOPT_USERAGENT => VV_UPTIME_UA,
|
||||
CURLOPT_RETURNTRANSFER => true,
|
||||
// These are this host's own certificates behind its own proxy. Verification is left on
|
||||
// because a cert that stopped validating is exactly the failure worth catching.
|
||||
CURLOPT_SSL_VERIFYPEER => true,
|
||||
CURLOPT_SSL_VERIFYHOST => 2,
|
||||
]);
|
||||
curl_multi_add_handle($mh, $ch);
|
||||
$handles[$d] = $ch;
|
||||
}
|
||||
$t0 = microtime(true);
|
||||
$running = null;
|
||||
do {
|
||||
curl_multi_exec($mh, $running);
|
||||
if ($running) curl_multi_select($mh, 1.0);
|
||||
} while ($running);
|
||||
|
||||
$upN = $downN = 0;
|
||||
$downList = [];
|
||||
$changes = [];
|
||||
$doms = $store['domains'] ?? [];
|
||||
$hourKey = date('YmdH', $now);
|
||||
$dayKey = date('Ymd', $now);
|
||||
$monKey = date('Ym', $now);
|
||||
|
||||
foreach ($handles as $d => $ch) {
|
||||
$errno = curl_errno($ch);
|
||||
$code = (int) curl_getinfo($ch, CURLINFO_HTTP_CODE);
|
||||
$ms = (int) round(curl_getinfo($ch, CURLINFO_TOTAL_TIME) * 1000);
|
||||
$err = curl_error($ch);
|
||||
curl_multi_remove_handle($mh, $ch);
|
||||
curl_close($ch);
|
||||
|
||||
$up = ($errno === 0 && $code > 0 && $code < 500);
|
||||
$detail = $errno !== 0 ? ($err ?: 'connection failed') : ('HTTP ' . $code);
|
||||
$up ? $upN++ : $downN++;
|
||||
if (!$up) $downList[] = "$d — $detail";
|
||||
|
||||
$r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null,
|
||||
'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [],
|
||||
'months' => [], 'events' => [], 'since' => $now];
|
||||
$r['checks']++;
|
||||
$up ? $r['up']++ : $r['down']++;
|
||||
$r['last_code'] = $code;
|
||||
$r['last_ms'] = $ms;
|
||||
$r['last_at'] = $now;
|
||||
$r['last_detail'] = $detail;
|
||||
|
||||
$newState = $up ? 'up' : 'down';
|
||||
if (($r['state'] ?? null) !== $newState) {
|
||||
// The first observation is not a transition — there was no previous state to leave.
|
||||
if ($r['state'] !== null) {
|
||||
$ev = ['ts' => $now, 'to' => $newState, 'detail' => $detail];
|
||||
$r['events'][] = $ev;
|
||||
$r['events'] = array_slice($r['events'], -VV_EVENTS_KEEP);
|
||||
$changes[] = "$d → " . strtoupper($newState) . " ($detail)";
|
||||
}
|
||||
$r['state'] = $newState;
|
||||
$r['last_change'] = $now;
|
||||
}
|
||||
|
||||
$r['samples'][] = $up ? 1 : 0;
|
||||
if (count($r['samples']) > VV_SAMPLES_KEEP)
|
||||
$r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP);
|
||||
|
||||
foreach ([['hours', $hourKey, VV_HOURS_KEEP],
|
||||
['days', $dayKey, VV_DAYS_KEEP],
|
||||
['months', $monKey, VV_MONTHS_KEEP]] as [$k, $key, $keep]) {
|
||||
$b = $r[$k][$key] ?? ['u' => 0, 't' => 0];
|
||||
$b['t']++;
|
||||
if ($up) $b['u']++;
|
||||
$r[$k][$key] = $b;
|
||||
if (count($r[$k]) > $keep) { krsort($r[$k]); $r[$k] = array_slice($r[$k], 0, $keep, true); ksort($r[$k]); }
|
||||
}
|
||||
|
||||
$doms[$d] = $r;
|
||||
}
|
||||
curl_multi_close($mh);
|
||||
|
||||
// Retire domains nothing has probed in a long time. Keyed on last_at rather than on absence
|
||||
// from the current list, which matters when NPM is unreachable: the cached list keeps being
|
||||
// probed, every domain keeps getting a last_at, and an NPM outage therefore cannot empty the
|
||||
// store. Only a domain that genuinely left the list stops being stamped.
|
||||
$cutoff = $now - (VV_DOMAIN_DROP_DAYS * 86400);
|
||||
$dropped = [];
|
||||
foreach ($doms as $d => $r) {
|
||||
// A record with no last_at at all is kept. It should not be possible — every probe stamps
|
||||
// it — and deleting on missing data is the wrong way round for something irreversible.
|
||||
if (isset($r['last_at']) && $r['last_at'] < $cutoff) { unset($doms[$d]); $dropped[] = $d; }
|
||||
}
|
||||
|
||||
$store['domains'] = $doms;
|
||||
$store['last_pass'] = $now;
|
||||
|
||||
printf("%d domains — %d up, %d down, %.1fs%s\n", count($handles), $upN, $downN,
|
||||
microtime(true) - $t0, $changes ? '' : ' (no state changes)');
|
||||
foreach ($changes as $c) echo " $c\n";
|
||||
foreach ($downList as $c) echo " DOWN $c\n";
|
||||
// Named, not silent. Dropping a record throws away months of history, and a line in the log is
|
||||
// the only trace that it was this and not the store being reset by something else.
|
||||
foreach ($dropped as $d)
|
||||
echo " RETIRED $d — not probed in " . VV_DOMAIN_DROP_DAYS . " days\n";
|
||||
|
||||
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
||||
if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); }
|
||||
exit(0);
|
||||
|
||||
} finally {
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
}
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ==================================== Uptime Probe ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Probes every hostname NPM serves, from outside the proxy, once per run. Produces the uptime
|
||||
# percentages and history strip on the Proxies tab, and the wobble list for the Sunday report.
|
||||
#
|
||||
# NPM's access log only describes hosts somebody visited. The host most likely to be quietly
|
||||
# broken is the one nobody visited, and nothing was watching those at all.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# A wrapper. The work is in uptime_probe.php, which issues all probes in parallel through
|
||||
# curl_multi so one pass costs about as long as the slowest domain rather than the sum.
|
||||
#
|
||||
# Runs every minute, injected by include/scheduler.php alongside the other background writers.
|
||||
# Nothing about the storage assumes that cadence — everything is counts and time buckets.
|
||||
#
|
||||
# Probes carry the User-Agent Varaverk-Uptime/1.0, which npm_access_stats.sh excludes. Without
|
||||
# that, this monitor becomes fifty thousand requests a day in the logs it reports on.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Probe what is served, not what was visited.
|
||||
# The domain list comes from NPM's own proxy hosts, so a host nobody has requested is measured
|
||||
# exactly like a busy one. Deriving the list from traffic would leave the quietest hosts — the
|
||||
# ones most likely to be broken without anyone noticing — permanently unmonitored.
|
||||
#
|
||||
# The probe is excluded from the statistics it feeds.
|
||||
# Every request carries Varaverk-Uptime/1.0, which npm_access_stats.sh filters out. Without
|
||||
# that the monitor would be the largest single source of traffic in the logs it reports on,
|
||||
# and every access figure would be measuring this script.
|
||||
#
|
||||
# The wrapper holds no logic.
|
||||
# Flags are forwarded verbatim and nothing is interpreted here. Two places that both know what
|
||||
# --dry-run means is two places that can disagree about it.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# UPTIME_PROBE_ENABLED gates the whole run. Off means no probes and no writes.
|
||||
#
|
||||
# Bounded per domain by UPTIME_PROBE_TIMEOUT, so an unresponsive host costs one timeout rather
|
||||
# than stalling the pass — the probes run in parallel, so one slow domain never delays the rest.
|
||||
#
|
||||
# --dry-run probes and reports without writing. --status and --events read stored history and
|
||||
# probe nothing, so neither can alter what it is describing.
|
||||
#
|
||||
# Read-only against NPM. The domain list is read; no proxy host, certificate or setting is
|
||||
# touched by anything in this path.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# uptime_probe.sh one pass
|
||||
# uptime_probe.sh --dry-run probe and report, write nothing
|
||||
# uptime_probe.sh --status per-domain uptime table (24h / 7d / 30d / 1y)
|
||||
# uptime_probe.sh --events recent state changes, newest first
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# UPTIME_PROBE_ENABLED master switch
|
||||
# UPTIME_PROBE_TIMEOUT seconds per domain
|
||||
# UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list from NPM
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
php "$SCRIPT_DIR/uptime_probe.php" "$@"
|
||||
+511
-7
@@ -8,7 +8,442 @@ $docroot = $docroot ?? $_SERVER['DOCUMENT_ROOT'] ?: '/usr/local/emhttp';
|
||||
$pluginDir = "$docroot/plugins/$plugin";
|
||||
|
||||
require_once "$pluginDir/include/config.php";
|
||||
?>
|
||||
|
||||
<script>
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// CSRF token propagation — must run before any page JS.
|
||||
//
|
||||
// Unraid enforces CSRF centrally: /etc/php.ini sets auto_prepend_file to webGui's
|
||||
// local_prepend.php, which terminates *every* POST that does not carry a valid token, before a
|
||||
// single line of endpoint code runs. It accepts the token as a `csrf_token` POST field or as an
|
||||
// X-CSRF-Token header.
|
||||
//
|
||||
// The webGUI's own injector is jQuery-only ($.ajaxPrefilter in HeadInlineJS.php). Varaverk's
|
||||
// pages use native fetch(), which that prefilter does not touch — so without this shim every
|
||||
// mutating request in the plugin is silently killed by the platform. Silently, because
|
||||
// csrf_terminate() exits with no body: the fetch resolves, r.json() throws on the empty
|
||||
// response, and the page's own .catch() swallows it.
|
||||
//
|
||||
// Setting the header rather than appending a body field is deliberate. It works identically for
|
||||
// FormData, URLSearchParams and raw JSON bodies, so no call site has to know about it and a new
|
||||
// endpoint cannot forget to include it. The header is only ever attached to same-origin
|
||||
// Varaverk URLs; a cross-origin page cannot set a custom header without a preflight it will
|
||||
// fail, which is precisely what makes this a CSRF defence rather than a formality.
|
||||
//
|
||||
// Inline, and above the tab content, because pages/*.php carry their own inline fetch calls and
|
||||
// some fire on load — an external script could not be guaranteed to install first.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
(function () {
|
||||
if (window.__vvCsrfInstalled) return;
|
||||
window.__vvCsrfInstalled = true;
|
||||
|
||||
var nativeFetch = window.fetch.bind(window);
|
||||
|
||||
window.fetch = function (input, init) {
|
||||
var url = (typeof input === 'string') ? input : (input && input.url) || '';
|
||||
|
||||
if (url.indexOf('/plugins/varaverk/') !== -1 &&
|
||||
typeof csrf_token !== 'undefined' && csrf_token) {
|
||||
init = init || {};
|
||||
var headers = new Headers(init.headers || (typeof input === 'object' && input.headers) || {});
|
||||
if (!headers.has('X-CSRF-Token')) headers.set('X-CSRF-Token', csrf_token);
|
||||
init = Object.assign({}, init, { headers: headers });
|
||||
}
|
||||
|
||||
return nativeFetch(input, init);
|
||||
};
|
||||
})();
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// Escaping helpers — shared, because every page builds HTML strings and assigns them to innerHTML.
|
||||
//
|
||||
// Defined here rather than per page for the reason the CSRF shim is: only one pages/*.php is ever
|
||||
// included per request, so a helper defined inside one page does not exist for any other. That is
|
||||
// not a hypothetical — these lived in pages/scheduler.php, and pages/monitor.php rendered media
|
||||
// titles, partner hostnames and docker folder names straight into innerHTML with no escaping
|
||||
// available to it at all.
|
||||
//
|
||||
// Not in js/varaverk.js, which would otherwise be the obvious home: that file is loaded by a
|
||||
// src-tag *below* the tab include, so it is not defined yet while a page's inline script is
|
||||
// running. The shared formatters there survive only because every caller is inside a fetch
|
||||
// callback. An escaping helper must be callable from the first synchronous line of a page.
|
||||
//
|
||||
// Never write a literal script tag inside this block, not even in a comment. The tab body is
|
||||
// injected into the page by the webGUI rather than parsed as part of the document, and the
|
||||
// opening sequence ends this element early wherever it appears — the whole file then renders as
|
||||
// visible text and every tab loads dead. That is not theoretical: the comment above said it with
|
||||
// real angle brackets and took the monitor tab down on 2026-08-07.
|
||||
//
|
||||
// String() rather than assuming a string: these are fed payload fields that are frequently
|
||||
// numbers, and sometimes null or undefined. A helper that throws on a number is a helper call
|
||||
// sites will skip.
|
||||
//
|
||||
// Two functions because the contexts differ, and using the wrong one is silent:
|
||||
// vvEscHtml text between tags. Leaves " alone — harmless there.
|
||||
// vvEscAttr text inside an attribute. Escapes " as well, because a quote inside a
|
||||
// double-quoted attribute ends it early and destroys the rest of the handler.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
function vvEscHtml(s) {
|
||||
return String(s ?? '').replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
|
||||
}
|
||||
function vvEscAttr(s) {
|
||||
return String(s ?? '').replace(/&/g,'&').replace(/"/g,'"')
|
||||
.replace(/</g,'<').replace(/>/g,'>');
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// What a failed fetch says.
|
||||
//
|
||||
// Every fetch in this plugin used to end in `.catch(() => {})` — 24 of them. That is not error
|
||||
// handling, it is error deletion: the request fails, nothing renders, nothing is logged, and the
|
||||
// surface either sits on "Loading…" forever or silently keeps showing stale numbers. The mesh chat
|
||||
// spent an unknown amount of time "taking a minute to load" because a ReferenceError was thrown on
|
||||
// every render and swallowed here; the fault named itself the moment a catch reported it.
|
||||
//
|
||||
// Console always, because a poller that drops one tick should not shout on screen. A target
|
||||
// element when the caller has one, because a panel that will otherwise never fill has to say why.
|
||||
//
|
||||
// Global for the same reason vvEscHtml is: pages/*.php are included one at a time and each would
|
||||
// otherwise carry its own copy, which is the arrangement that lets two of them drift.
|
||||
function vvFetchErr(where, e, el) {
|
||||
const msg = (e && e.message) ? e.message : String(e || 'request failed');
|
||||
try { console.warn('[varaverk] ' + where + ' — ' + msg, e); } catch (_) {}
|
||||
if (el) {
|
||||
const n = (typeof el === 'string') ? document.getElementById(el) : el;
|
||||
if (n) { n.textContent = where + ' failed: ' + msg; n.style.color = '#a05a2c'; }
|
||||
}
|
||||
}
|
||||
|
||||
// Throws on a non-2xx instead of handing HTML to JSON.parse. Unraid answers an expired session
|
||||
// with a 302 to the login page, so without this the reported error is "Unexpected token '<'",
|
||||
// which names the symptom and hides the cause.
|
||||
function vvJson(r) {
|
||||
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||||
return r.json();
|
||||
}
|
||||
|
||||
// A URL about to be put in href/src or handed to window.open. Anything that is not plainly http,
|
||||
// https or a site-relative path becomes empty — javascript: is the one that matters, and an
|
||||
// allowlist is the only way to say that without chasing encodings. include/docs.php applies the
|
||||
// same rule to markdown links; container WebUI values, which come from template XML, had no such
|
||||
// check before reaching window.open().
|
||||
function vvSafeUrl(u) {
|
||||
const s = String(u ?? '').trim();
|
||||
if (s === '') return '';
|
||||
if (/^https?:\/\//i.test(s)) return s;
|
||||
if (/^\/(?!\/)/.test(s)) return s;
|
||||
return '';
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// DIALOGS — never the browser's own
|
||||
//
|
||||
// confirm(), alert() and prompt() all carry a checkbox inside the dialog reading "prevent this
|
||||
// page from creating additional dialogs". The moment it is ticked, every later call from that
|
||||
// document returns false and draws nothing. Guards written as `if (!confirm(x)) return;` then
|
||||
// decline silently and the button reads as dead — and because the webGUI swaps tabs by AJAX
|
||||
// without tearing down the document, the suppression follows the operator across tabs and
|
||||
// survives until a full reload. That happened on 2026-08-09: dismissing several repair findings
|
||||
// in a row produced the checkbox, and the AI tab's buttons went dead until history was cleared.
|
||||
//
|
||||
// swal is Unraid's own dialog, defined in webGui/javascript/dynamix.js and loaded on every page
|
||||
// by DefaultPageLayout — core calls it around 370 times and uses confirm() exactly never. So
|
||||
// this is not a new dependency, and these dialogs look like the rest of the machine.
|
||||
//
|
||||
// THE TRAP, and why the callback below is not an arrow function:
|
||||
// On cancel, SweetAlert 1.x only invokes the callback if the callback's own source declares a
|
||||
// parameter — it literally does String(fn).replace(/\s/g,'') and checks that it starts with
|
||||
// "function(" and that the next character is not ")". An arrow function stringifies as "ok=>…"
|
||||
// and fails that test, so cancel would never resolve this promise and the awaiting caller
|
||||
// would hang forever with no dialog on screen. Which is the exact bug this file is replacing,
|
||||
// wearing a different hat. Classic function expression, one named parameter, deliberately.
|
||||
function vvConfirm(text, opts) {
|
||||
const o = opts || {};
|
||||
return new Promise(function (resolve) {
|
||||
// Without swal there is nothing better than the browser's own dialog. It carries the bug
|
||||
// described above, but the alternative is a guard that answers neither yes nor no.
|
||||
if (typeof swal !== 'function') { resolve(window.confirm(text)); return; }
|
||||
swal({
|
||||
title: o.title || 'Are you sure?',
|
||||
text: String(text ?? ''),
|
||||
type: o.type || 'warning',
|
||||
showCancelButton: true,
|
||||
confirmButtonText: o.confirmText || 'Yes',
|
||||
cancelButtonText: o.cancelText || 'Cancel',
|
||||
confirmButtonColor: o.danger === false ? '#3085d6' : '#d9534f',
|
||||
closeOnConfirm: true,
|
||||
}, function (ok) { resolve(ok !== false); });
|
||||
});
|
||||
}
|
||||
|
||||
// Telling the operator something, with no question attached. Returns a promise so a caller can
|
||||
// sequence on it, but nothing has to await it — which is what let the fifty-odd alert() sites
|
||||
// become this by rename alone, with no function above them turning async.
|
||||
//
|
||||
// The icon is read off the message when the caller does not say. Almost every one of these
|
||||
// reports a failure — "Save failed: …", "Error: …" — and asking fifty call sites to each classify
|
||||
// themselves would mean fifty chances to disagree about what counts as an error. The caller can
|
||||
// still pass type explicitly and that always wins.
|
||||
function vvAlertType(text) {
|
||||
const s = String(text ?? '');
|
||||
if (/\b(fail|failed|error|denied|invalid|refused|cannot|could not|unable)\b/i.test(s)) return 'error';
|
||||
if (/\b(warn|warning|already|must be)\b/i.test(s)) return 'warning';
|
||||
return 'info';
|
||||
}
|
||||
function vvAlert(text, opts) {
|
||||
const o = opts || {};
|
||||
return new Promise(function (resolve) {
|
||||
if (typeof swal !== 'function') { window.alert(text); resolve(true); return; }
|
||||
swal({
|
||||
title: o.title || '',
|
||||
text: String(text ?? ''),
|
||||
type: o.type || vvAlertType(text),
|
||||
confirmButtonText: o.confirmText || 'OK',
|
||||
}, function (ok) { resolve(true); });
|
||||
});
|
||||
}
|
||||
|
||||
// Asking for a value. Resolves to the string, or null when cancelled — prompt()'s own contract,
|
||||
// so call sites keep reading the same way. The empty string is a real answer and is not null.
|
||||
function vvPrompt(text, def, opts) {
|
||||
const o = opts || {};
|
||||
return new Promise(function (resolve) {
|
||||
if (typeof swal !== 'function') { resolve(window.prompt(text, def || '')); return; }
|
||||
swal({
|
||||
title: o.title || '',
|
||||
text: String(text ?? ''),
|
||||
type: 'input',
|
||||
inputValue: def || '',
|
||||
inputPlaceholder: o.placeholder || '',
|
||||
showCancelButton: true,
|
||||
confirmButtonText: o.confirmText || 'OK',
|
||||
cancelButtonText: o.cancelText || 'Cancel',
|
||||
closeOnConfirm: true,
|
||||
}, function (val) { resolve(val === false ? null : String(val)); });
|
||||
});
|
||||
}
|
||||
|
||||
// ── Mirror onboarding, shared by the Partnership tab and the first-run wizard ─────────────────
|
||||
//
|
||||
// Here for the reason stated above: only one pages/*.php is included per request, and both the
|
||||
// wizard and the Partnership tab need to render the same panel and run the same job. A second
|
||||
// copy in setup.php would drift from the one in partnership.php, and the panel encodes a detail
|
||||
// that is easy to get wrong on a copy — the terminal command must carry the SERVING host's
|
||||
// SCRIPTS_DIR, because it is pasted into a terminal on that machine.
|
||||
//
|
||||
// Not in js/varaverk.js: that loads below the tab include, and the wizard returns before it.
|
||||
|
||||
// Where Varaverk is installed on THIS host, read live rather than baked into the page.
|
||||
//
|
||||
// Two reasons it cannot be a render-time constant. The wizard can move it: choosing appdata in
|
||||
// step 1 triggers a migration and step 2 renders in the same page load, so PHP's value names the
|
||||
// pre-migration location. And each host chooses independently — the owner may be on flash while
|
||||
// the mirror is on appdata — so a panel rendered on either side must ask, not assume.
|
||||
//
|
||||
// It matters because the value ends up in a command the operator pastes into a root terminal.
|
||||
// api/setup.php?action=detect re-reads varaverk.cfg. Cached; the panels re-render often.
|
||||
let _vvScriptsDir = null;
|
||||
function vvScriptsDir(fallback) {
|
||||
if (_vvScriptsDir) return Promise.resolve(_vvScriptsDir);
|
||||
return fetch('/plugins/varaverk/api/setup.php?action=detect&_=' + Date.now())
|
||||
.then(r => r.json())
|
||||
.then(d => (_vvScriptsDir = (d && d.scripts_dir) || fallback))
|
||||
.catch(() => (_vvScriptsDir = fallback));
|
||||
}
|
||||
|
||||
// api/run.php answers when a job is LAUNCHED, not finished. An empty body is never success —
|
||||
// Unraid's CSRF guard exits with one, and so does a PHP fatal.
|
||||
function _vvPtRun(id, extraArgs) {
|
||||
const params = {id, manual: '1'};
|
||||
if (extraArgs) params.extra_args = extraArgs;
|
||||
return fetch('/plugins/varaverk/api/run.php', {method: 'POST', body: new URLSearchParams(params)})
|
||||
.then(r => { if (!r.ok) throw new Error('HTTP ' + r.status); return r.text(); })
|
||||
.then(text => {
|
||||
if (!text.trim()) throw new Error('Empty response — request rejected before it reached run.php');
|
||||
return JSON.parse(text);
|
||||
});
|
||||
}
|
||||
|
||||
function _vvJobProgressEl(btn) {
|
||||
let el = btn.parentElement.querySelector('.vv-jobprog');
|
||||
if (!el) {
|
||||
el = document.createElement('div');
|
||||
el.className = 'vv-jobprog';
|
||||
el.style.cssText = 'font-size:10px;margin-top:5px;white-space:nowrap;';
|
||||
btn.parentElement.appendChild(el);
|
||||
}
|
||||
el.innerHTML = '<span style="color:#4a9eff;">⟳ starting…</span>';
|
||||
return el;
|
||||
}
|
||||
|
||||
// Polls the job's stat for liveness and its log for the step banner, so a launched job shows
|
||||
// where it is instead of appearing to do nothing for minutes.
|
||||
const _vvJobPoll = {};
|
||||
function vvPtWatchJob(id, mountEl) {
|
||||
if (_vvJobPoll[id]) clearInterval(_vvJobPoll[id]);
|
||||
const enc = encodeURIComponent(id);
|
||||
const tick = () => {
|
||||
fetch(`/plugins/varaverk/api/status.php?id=${enc}&_=${Date.now()}`)
|
||||
.then(r => r.json())
|
||||
.then(s => {
|
||||
if (!s.ok) return;
|
||||
if (s.status === 'running') {
|
||||
return fetch(`/plugins/varaverk/api/log.php?id=${enc}&_=${Date.now()}`)
|
||||
.then(r => r.json())
|
||||
.then(l => {
|
||||
const lines = (l.content || '').split('\n');
|
||||
let step = '';
|
||||
for (let i = lines.length - 1; i >= 0; i--) {
|
||||
const m = lines[i].match(/━━━\s*(?:[^\s]+\s+)?(Step [^━]+?)\s*━━━/);
|
||||
if (m) { step = m[1].trim(); break; }
|
||||
}
|
||||
mountEl.innerHTML = `<span style="color:#4a9eff;">⟳ running</span>`
|
||||
+ (step ? ` <span style="color:#666;">· ${vvEscHtml(step)}</span>` : '');
|
||||
});
|
||||
}
|
||||
clearInterval(_vvJobPoll[id]); delete _vvJobPoll[id];
|
||||
const col = s.status === 'ok' ? '#4caf50' : (s.status === 'warn' ? '#ff9800' : '#f44336');
|
||||
const lbl = s.status === 'ok' ? 'complete ✅'
|
||||
: (s.status === 'never_run' ? 'did not start ⚠' : `${s.status} (exit ${s.exit ?? '?'})`);
|
||||
mountEl.innerHTML = `<span style="color:${col};">${lbl}</span>`
|
||||
+ ` <a href="?tab=scheduler" class="localURL" style="color:#556;margin-left:6px;">log</a>`;
|
||||
if (typeof _vvPtReload === 'function') _vvPtReload();
|
||||
// The wizard does have something to refresh, and saying it did not is why the mirror's
|
||||
// Join button stayed greyed at "Running" after the run finished. Worse, the mirror's own
|
||||
// job takes about three seconds — it only notifies the owner — while the Phase 2 it
|
||||
// triggers runs for minutes on the far side. So "this job is done" is not "the
|
||||
// partnership is done", and a single reload here would still show an unfinished wizard.
|
||||
// vvOnJobDone polls until the checklist actually turns.
|
||||
if (typeof vvOnJobDone === 'function') vvOnJobDone(id, s);
|
||||
})
|
||||
.catch(() => {});
|
||||
};
|
||||
tick();
|
||||
_vvJobPoll[id] = setInterval(tick, 4000);
|
||||
}
|
||||
|
||||
async function vvPtOnboard(btn) {
|
||||
if (!await vvConfirm('Run full partnership_onboard.sh?\n\nRun on the MIRROR first, then on the OWNER.\n\nUse Phase 1 + Phase 2 buttons for step-by-step control.')) return;
|
||||
btn.disabled = true;
|
||||
btn.textContent = '⟳ Starting…';
|
||||
// Stays disabled while it runs. A timed re-enable invited the second click whose lock refusal
|
||||
// overwrote the live run's job record.
|
||||
const prog = _vvJobProgressEl(btn);
|
||||
_vvPtRun('Partnership/partnership_onboard.sh')
|
||||
.then(d => {
|
||||
if (!d.ok) throw new Error(d.error ?? 'Unknown error');
|
||||
btn.textContent = '⟳ Running…';
|
||||
vvPtWatchJob('Partnership/partnership_onboard.sh', prog);
|
||||
})
|
||||
.catch(e => {
|
||||
prog.innerHTML = `<span style="color:#f44336;">failed to start — ${vvEscHtml(String(e.message || e))}</span>`;
|
||||
btn.disabled = false;
|
||||
btn.textContent = '▶ Onboard';
|
||||
});
|
||||
}
|
||||
|
||||
// The mirror's two-step join. Step 1 is a terminal step on purpose: ssh_setup.sh runs ssh-copy-id,
|
||||
// which prompts for the owner's root password on a first install, and a WebGUI button cannot
|
||||
// answer a password prompt. Offering only a button here was offering the one route that cannot
|
||||
// work — it failed on ssh-copy-id every time.
|
||||
// opts: {ownerName, termBase, termCmd, phase, hasPartner}
|
||||
function vvRenderMirrorOnboard(opts) {
|
||||
const owner = vvEscHtml(opts.ownerName || 'the owner');
|
||||
const dis = opts.hasPartner === false ? 'disabled style="opacity:.35;cursor:default;"' : '';
|
||||
|
||||
// Collapses at phase 2, not phase 1.
|
||||
//
|
||||
// HOST<n>_PHASE1_DONE means the OWNER finished its Phase 1 — conf pushed, network created,
|
||||
// confs cached. It says nothing about whether THIS host's key was ever installed on the owner,
|
||||
// which is the only thing Step 1 does. Gating on it hid the terminal command at precisely the
|
||||
// moment the mirror still needed it: flag set by the owner's push, key not installed, panel
|
||||
// showing a button that cannot work.
|
||||
//
|
||||
// Phase 2 means the partnership is actually established, so the instructions have genuinely
|
||||
// stopped being needed. Until then showing them costs nothing — a step already done reads as a
|
||||
// reminder, a step still needed and hidden is a dead end.
|
||||
// Phase 2 runs on the OWNER and takes minutes — 4m05s on a measured run, longer as the arr
|
||||
// library grows. The mirror's own job finishes in about three seconds, because all it does is
|
||||
// send the notification, so for the rest of that window the screen showed the unchanged
|
||||
// phase-1 panel with a greyed-out button and no statement that anything was happening
|
||||
// elsewhere. That reads as a hang, and the reasonable response to a hang is to start clicking.
|
||||
if ((opts.phase ?? 0) < 2 && opts.waiting) {
|
||||
const mins = opts.waitingSince
|
||||
? Math.floor((Date.now() - opts.waitingSince) / 60000) : 0;
|
||||
const secs = opts.waitingSince
|
||||
? Math.floor(((Date.now() - opts.waitingSince) % 60000) / 1000) : 0;
|
||||
return `<div style="padding:12px 14px;background:#1a1200;border:1px solid #3a2800;border-radius:4px;">
|
||||
<div style="display:flex;align-items:center;gap:10px;flex-wrap:wrap;">
|
||||
<span style="font-size:18px;line-height:1;color:#ff9800;">⟳</span>
|
||||
<div style="flex:1;min-width:220px;">
|
||||
<div style="color:#ff9800;font-size:12px;font-weight:600;">
|
||||
Onboard in progress — please wait</div>
|
||||
<div style="color:#8a6a2a;font-size:10px;margin-top:3px;line-height:1.5;">
|
||||
${owner} is running Phase 2: deploying containers, syncing auth data and
|
||||
bootstrapping the arr libraries. This normally takes 4–6 minutes and nothing is
|
||||
needed from you here. This page updates itself when it finishes.</div>
|
||||
${opts.waitingSince ? `<div style="color:#6a5020;font-size:10px;margin-top:4px;
|
||||
font-family:monospace;">elapsed ${mins}m ${String(secs).padStart(2,'0')}s</div>` : ''}
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
if ((opts.phase ?? 0) >= 2) {
|
||||
// Done state. This used to return the same "Join partnership · ▶ Onboard" panel as the
|
||||
// not-started state, under a comment saying it collapsed — so a fully onboarded mirror
|
||||
// rendered as one that had never run, and the only honest reading of the screen was that
|
||||
// nothing had happened. The button is gone rather than disabled: there is nothing left for
|
||||
// the mirror to initiate, and Phase 2 is re-run from the owner.
|
||||
return `<div style="padding:10px 12px;background:#0a0f0a;border:1px solid #1a3a1a;border-radius:4px;">
|
||||
<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">
|
||||
<span style="font-size:10px;font-weight:600;color:#4caf50;background:#0a1a0a;
|
||||
padding:2px 9px;border-radius:10px;border:1px solid #1a4a1a;">Partnership active</span>
|
||||
<span style="font-size:9px;color:#444;">onboarded by ${owner} · nothing further to do here</span>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
return `<div style="padding:10px 12px;background:#0d0d0d;border:1px solid #1e3a5a;border-radius:4px;">
|
||||
<div style="display:flex;flex-direction:column;gap:10px;">
|
||||
<div>
|
||||
<div style="display:flex;align-items:center;gap:6px;margin-bottom:6px;">
|
||||
<span style="font-size:9px;font-weight:700;color:#4a9eff;background:#0a1828;
|
||||
padding:2px 8px;border-radius:10px;border:1px solid #1a3a5a;">Step 1</span>
|
||||
<span style="font-size:11px;color:#888;">Install SSH key on ${owner}</span>
|
||||
</div>
|
||||
<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-bottom:4px;">
|
||||
<a href="${opts.termBase}" target="_blank" class="localURL"
|
||||
style="padding:3px 10px;background:#0e1a2a;color:#7ab;border:1px solid #1e3a5a;
|
||||
border-radius:3px;text-decoration:none;font-size:10px;white-space:nowrap;">Open Terminal</a>
|
||||
<code onclick="navigator.clipboard.writeText('${opts.termCmd}').then(()=>{this.style.color='#4caf50';setTimeout(()=>this.style.color='#444',1500)})"
|
||||
style="font-size:9px;color:#444;background:#080808;padding:3px 8px;border-radius:3px;
|
||||
border:1px solid #181818;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;
|
||||
white-space:nowrap;cursor:pointer;" title="Click to copy">${opts.termCmd}</code>
|
||||
</div>
|
||||
<div style="font-size:9px;color:#2a2a2a;">Enter ${owner} root password when prompted · a button cannot answer that prompt</div>
|
||||
</div>
|
||||
<div style="border-top:1px solid #1a1a1a;padding-top:10px;">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;gap:10px;">
|
||||
<div>
|
||||
<div style="display:flex;align-items:center;gap:6px;margin-bottom:2px;">
|
||||
<span style="font-size:9px;font-weight:700;color:#4caf50;background:#0a1a0a;
|
||||
padding:2px 8px;border-radius:10px;border:1px solid #1a3a1a;">Step 2</span>
|
||||
<span style="font-size:11px;color:#888;">Join partnership</span>
|
||||
</div>
|
||||
<div style="font-size:9px;color:#333;">Notifies ${owner} to run Phase 2 · if key already installed</div>
|
||||
</div>
|
||||
<button class="vv-pt-action-btn run" onclick="vvPtOnboard(this)" ${dis}
|
||||
style="font-size:11px;white-space:nowrap;">▶ Onboard</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>`;
|
||||
}
|
||||
</script>
|
||||
|
||||
<?php
|
||||
// First-run check — show setup wizard if HOST1 is blank OR local host.conf is missing
|
||||
$_master = vv_read_conf_raw('master.conf');
|
||||
preg_match('/^\s*HOST1\s*=\s*"([^"]*)"/m', $_master, $_h1m);
|
||||
@@ -24,12 +459,78 @@ unset($_master, $_h1m, $_host1_blank, $_my_hostid, $_conf_missing);
|
||||
|
||||
// Determine active tab
|
||||
$tab = $_GET['tab'] ?? 'monitor';
|
||||
$validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fallback', 'arrs', 'rsync', 'auth', 'settings'];
|
||||
$validTabs = ['monitor', 'scheduler', 'watchdog', 'partnership', 'fallback', 'arrs', 'rsync', 'auth', 'settings'];
|
||||
|
||||
// The AI tab exists only on the AI owner, and only while AI_ENABLED is true. Appended to
|
||||
// $validTabs rather than filtered out of it, so the check below rejects ?tab=ai server-side as
|
||||
// well — omitting the link is presentation, not access control, and api/ai.php refuses the
|
||||
// owner-only actions independently.
|
||||
//
|
||||
// The host half is a deliberate split, not a technical limit. The mesh shares one AI: every node
|
||||
// reaches the owner's model through include/ai_rpc.php, so an assistant works everywhere. What
|
||||
// does not travel is this tab — it carries the bug reports, the index and the model configuration,
|
||||
// the surface where a wrong answer is expensive and the vocabulary assumes you built the thing.
|
||||
// Someone running two containers on a node they were handed gets the assistant, not the machinery
|
||||
// behind it.
|
||||
//
|
||||
// Assistant docks elsewhere use vv_ai_ui_on(), which every node in the mesh passes.
|
||||
$_vv_ai = vv_ai_owner_ui_on();
|
||||
if ($_vv_ai) $validTabs[] = 'ai';
|
||||
|
||||
// ── Local pages ───────────────────────────────────────────────────────────────────────────────
|
||||
// pages/local/ is gitignored, so whatever is in it belongs to this installation alone and never
|
||||
// reaches the public mirror. This loader is the tracked half: a generic extension point that
|
||||
// knows nothing about what it is loading.
|
||||
//
|
||||
// It exists because the alternative — a tracked `if (file_exists(pages/thing.php))` per private
|
||||
// page — puts the name and purpose of every private page into the public repo, which defeats the
|
||||
// point of keeping the page out of it.
|
||||
//
|
||||
// Discovered rather than configured: a conf key listing local pages would itself be a tracked
|
||||
// file naming them, and an untracked one would be a second thing to keep in sync with the
|
||||
// directory. The directory is the declaration.
|
||||
//
|
||||
// The label comes from a `// vv-local-page: Name` line in the first 2KB of the file, falling back
|
||||
// to the capitalised id. Reading it out of the file keeps the page self-describing — nothing
|
||||
// outside it has to be edited to add one.
|
||||
$localPages = [];
|
||||
foreach (glob("$pluginDir/pages/local/*.php") ?: [] as $_lp) {
|
||||
$_id = basename($_lp, '.php');
|
||||
// Ids are restricted and collisions rejected: $tab is user input that becomes an include
|
||||
// path below, and a local page must never be able to shadow a real tab.
|
||||
if (!preg_match('/^[a-z0-9][a-z0-9_-]{0,31}$/', $_id)) continue;
|
||||
if (in_array($_id, $validTabs, true)) continue;
|
||||
$_lbl = ucfirst($_id);
|
||||
if (preg_match('/^\s*(?:\/\/|#)\s*vv-local-page:\s*(.+)$/m', (string)@file_get_contents($_lp, false, null, 0, 2048), $_m)) {
|
||||
$_lbl = trim($_m[1]);
|
||||
}
|
||||
$localPages[$_id] = ['path' => $_lp, 'label' => $_lbl];
|
||||
$validTabs[] = $_id;
|
||||
}
|
||||
unset($_lp, $_id, $_lbl, $_m);
|
||||
|
||||
if (!in_array($tab, $validTabs)) $tab = 'monitor';
|
||||
$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'Docker', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Arrs', 'rsync' => 'Rsync', 'auth' => 'Auth Stack', 'settings' => 'Settings'];
|
||||
$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Media Stack', 'rsync' => 'Rsync', 'auth' => 'Auth Stack', 'settings' => 'Settings', 'ai' => 'AI'];
|
||||
foreach ($localPages as $_id => $_lp) $tabLabels[$_id] = $_lp['label'];
|
||||
unset($_id, $_lp);
|
||||
|
||||
// Cache stamp for the stylesheet and script below. Both are served straight off the plugin
|
||||
// directory at a path that never changes, so a browser holding an old copy keeps using it after
|
||||
// a git pull. This page's own markup is generated fresh every request, so the two fall out of
|
||||
// step: markup from the new commit rendering against a stylesheet from the old one. That is not
|
||||
// a theoretical failure — it broke the Monitor AI row on 2026-08-09, when placement moved from
|
||||
// inline spans into the stylesheet and only half of that arrived in the browser.
|
||||
//
|
||||
// mtime rather than a hand-bumped version: it changes on exactly the event that matters, a pull
|
||||
// rewriting the file, and cannot be forgotten. Falling back to time() when the stat fails errs
|
||||
// toward re-fetching rather than toward the stale copy that caused the problem.
|
||||
$_vv_asset_rev = [];
|
||||
foreach (['css/varaverk.css', 'js/varaverk.js'] as $_vv_a) {
|
||||
$_vv_asset_rev[$_vv_a] = @filemtime(__DIR__ . '/' . $_vv_a) ?: time();
|
||||
}
|
||||
?>
|
||||
|
||||
<link rel="stylesheet" href="/plugins/<?=$plugin?>/css/varaverk.css">
|
||||
<link rel="stylesheet" href="/plugins/<?=$plugin?>/css/varaverk.css?v=<?=$_vv_asset_rev['css/varaverk.css']?>">
|
||||
|
||||
<div id="varaverk-wrap" class="unapi">
|
||||
|
||||
@@ -40,7 +541,7 @@ $tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'D
|
||||
<?= $tabLabels[$t] ?? ucfirst($t) ?>
|
||||
</a>
|
||||
<?php endforeach; ?>
|
||||
<div style="margin-left:auto;display:flex;align-items:center;gap:2px;">
|
||||
<div style="margin-left:auto;display:flex;align-items:center;gap:2px;flex-shrink:0;">
|
||||
<a href="https://github.com/FailedProxy/Varaverk" target="_blank"
|
||||
style="padding:0 10px;font-size:10px;color:#333;text-decoration:none;
|
||||
display:flex;align-items:center;letter-spacing:.03em;"
|
||||
@@ -54,12 +555,15 @@ $tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'D
|
||||
<!-- Tab content -->
|
||||
<div id="vv-content">
|
||||
<?php
|
||||
$page = "$pluginDir/pages/$tab.php";
|
||||
// Local pages resolve from the map built above, never by composing a path out of $tab —
|
||||
// the map's keys were validated against a strict pattern, so nothing user-supplied
|
||||
// reaches an include.
|
||||
$page = isset($localPages[$tab]) ? $localPages[$tab]['path'] : "$pluginDir/pages/$tab.php";
|
||||
if (file_exists($page)) include $page;
|
||||
else echo "<p>Page not found: $tab</p>";
|
||||
else echo "<p>Page not found: " . htmlspecialchars($tab, ENT_QUOTES) . "</p>";
|
||||
?>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="/plugins/<?=$plugin?>/js/varaverk.js"></script>
|
||||
<script src="/plugins/<?=$plugin?>/js/varaverk.js?v=<?=$_vv_asset_rev['js/varaverk.js']?>"></script>
|
||||
|
||||
@@ -186,7 +186,7 @@ log "WebGUI check — $WEBGUI_URL"
|
||||
|
||||
# ── Healthy — completely silent ───────────────────────────────────────────────────────────────
|
||||
if check_webgui; then
|
||||
_nginx_count=$(pgrep -cx nginx 2>/dev/null || echo 0)
|
||||
_nginx_count=$(pgrep -cx nginx 2>/dev/null || true)
|
||||
_fpm_count=$(pgrep -fc "php-fpm" 2>/dev/null || echo 0)
|
||||
log "$ICON_WEBGUI WebGUI responding ✅ — nginx workers:${_nginx_count} php-fpm workers:${_fpm_count}"
|
||||
echo "WebGUI responding — healthy ✅"
|
||||
|
||||
@@ -203,6 +203,46 @@ platform_get_temp_thresholds() {
|
||||
echo "${hdd_hot:-45} ${hdd_max:-55} ${ssd_hot:-60} ${ssd_max:-70}"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
# platform_get_cpu_temp
|
||||
# Writes the CPU temperature in °C to stdout as a decimal, or nothing if no sensor answers.
|
||||
# Returns 1 when nothing could be read, so a caller can tell "no sensor" from "cold".
|
||||
#
|
||||
# Two things make this harder than it looks, and both produced wrong numbers for months:
|
||||
#
|
||||
# 1. lm-sensors prints the alarm thresholds on the same line as the reading:
|
||||
# Core 0: +63.0 C (high = +80.0 C, crit = +100.0 C)
|
||||
# Anything that scrapes numbers off the whole line and takes the largest reports 100 —
|
||||
# the critical threshold — as the current temperature, on every Intel box, forever. The
|
||||
# parenthetical is stripped before a single digit is read.
|
||||
#
|
||||
# 2. On AMD, Tctl is not the die temperature. It is a control value carrying a fixed offset
|
||||
# (+27°C on Threadripper), which is why HOST1 read 70 while the die was at 43. Tdie is the
|
||||
# real measurement and is preferred wherever both are published.
|
||||
#
|
||||
# Preference order: Tdie, then Intel's package sensor, then the board's own CPU Temp, then the
|
||||
# hottest individual core. The first label that answers wins; within a label the maximum is
|
||||
# taken, because a multi-die part publishes one line per die and the hottest is the one that
|
||||
# matters.
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
platform_get_cpu_temp() {
|
||||
command -v sensors >/dev/null 2>&1 || return 1
|
||||
|
||||
local out label value
|
||||
out=$(sensors 2>/dev/null | sed 's/(.*//') # drop "(high = ..., crit = ...)"
|
||||
[[ -n "$out" ]] || return 1
|
||||
|
||||
for label in 'Tdie' 'Package id 0' 'CPU Temp' 'Core '; do
|
||||
value=$(echo "$out" | grep -i "^[[:space:]]*${label}" \
|
||||
| grep -oE '[+-]?[0-9]+\.[0-9]+' | sort -n | tail -1)
|
||||
if [[ -n "$value" ]]; then
|
||||
echo "${value#+}" # bash printf tolerates a leading +, PHP casts fine, awk does not
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
# platform_is_maintenance_running
|
||||
# Returns 0 if a parity check or sync is currently in progress.
|
||||
@@ -412,7 +452,12 @@ require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
|
||||
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
|
||||
\$failed = 0;
|
||||
foreach (\$results as \$r) {
|
||||
echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
|
||||
// 'seed' means the partner had no master.conf anywhere — say so, because that is the case
|
||||
// where the push carried the mesh's identity to a node that could not have known it.
|
||||
\$what = (\$r['mode'] ?? '') === 'seed'
|
||||
? 'seeded → ' . (\$r['path'] ?? '')
|
||||
: 'pushed → ' . (\$r['path'] ?? '');
|
||||
echo \$r['host'] . ': ' . (\$r['ok'] ? \$what : 'FAILED — ' . \$r['error']) . PHP_EOL;
|
||||
if (!\$r['ok']) \$failed++;
|
||||
}
|
||||
exit(\$failed > 0 ? 1 : 0);
|
||||
@@ -424,10 +469,14 @@ exit(\$failed > 0 ? 1 : 0);
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
# platform_push_setup_state
|
||||
# Pushes the Varaverk wizard setup state to all partners via the WebGUI PHP API.
|
||||
# No-op if php is unavailable. Always returns 0.
|
||||
# Returns 1 if any partner did not take it, 0 otherwise. No-op returning 0 if php is absent.
|
||||
#
|
||||
# This used to end with a bare `return 0` under the comment "Always returns 0", which meant
|
||||
# every caller that wrote `platform_push_setup_state || X=false` was testing a constant.
|
||||
# Wiring a failure branch onto a helper that cannot fail is worse than hardcoding the tick,
|
||||
# because the code reads as though it checked.
|
||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||
platform_push_setup_state() {
|
||||
command -v php &>/dev/null || return 0
|
||||
php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null
|
||||
return 0
|
||||
php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; exit(vv_push_setup_state() > 0 ? 1 : 0);" 2>/dev/null
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Browser entry point for the AI subsystem. Serves the status banner, starts a chat turn, and
|
||||
// reports its progress — the token-and-poll contract behind the AI page and every assistant
|
||||
// dock in the plugin.
|
||||
//
|
||||
// Transport only. The actions themselves live in include/ai_actions.php, which this shares with
|
||||
// Tools/ai_rpc.php — the mesh entry point another node reaches over SSH. This file owns the
|
||||
// things that are true of a browser request and nothing else: the CSRF-covered method split,
|
||||
// the request trace, the master switch, and the routing decision.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Generation takes 25-76 seconds on this hardware, so a turn is not answered in the request
|
||||
// that starts it. POST action=ask spawns Tools/ai_chat_worker.php detached and returns a
|
||||
// token immediately; the page polls action=poll until the job file reaches a terminal state.
|
||||
// This is the same shape api/manual_sync.php uses for long rsync runs, chosen over SSE so
|
||||
// the whole api layer keeps one response convention and one error path.
|
||||
//
|
||||
// Conversation history is capped here, not in the worker. The model is fully offloaded only
|
||||
// at 16384 context, and retrieved chunks plus reasoning already consume several thousand
|
||||
// tokens — unbounded history would silently cross that ceiling mid-conversation and cost
|
||||
// roughly 4x throughput. One place owns that policy.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Reads are GET, work is POST.
|
||||
// stats and poll change nothing and are safe to repeat. ask spawns a process, so it is
|
||||
// POST and therefore covered by Unraid's CSRF prepend, which inspects no GET at all.
|
||||
//
|
||||
// The banner is a separate action from the chat.
|
||||
// It is polled on a slow cycle and must keep rendering while a turn is in flight, so it
|
||||
// shares no state with the job.
|
||||
//
|
||||
// Tokens are minted here and never accepted from elsewhere.
|
||||
// random_bytes, hex, fixed length. A job's answer is readable by anyone who can guess
|
||||
// its token, so the token is not guessable.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// The per-action safeguards live with the actions.
|
||||
// Token minting, hex path validation, per-message history validation, scope whitelisting
|
||||
// and the detached spawn are all in include/ai_actions.php, documented there, and apply
|
||||
// identically to a browser request and a mesh request. Restating them here would be two
|
||||
// copies to keep in step and one of them always losing.
|
||||
//
|
||||
// AI_ENABLED is checked before routing, not after.
|
||||
// It is this node's own switch. A mirror with AI off must not forward to the owner —
|
||||
// honouring the toggle locally while quietly using someone else's model is not what the
|
||||
// switch says it does.
|
||||
//
|
||||
// Refusal is a 404, not a redirect or an empty 200.
|
||||
// The owner-only actions answer 404 off the owner. Omitting a link is presentation; this
|
||||
// endpoint is reachable directly, so the gate is enforced server-side too.
|
||||
//
|
||||
// A remote failure is reported as a remote failure.
|
||||
// vv_ai_rpc() returns the owner's own JSON when it gets one and a named transport error
|
||||
// when it does not. Neither is silently turned into an empty success — a mirror that
|
||||
// cannot reach the owner must say so rather than render an empty banner.
|
||||
//
|
||||
// REQUEST
|
||||
// GET ?action=stats banner payload
|
||||
// GET ?action=poll&token=<hex32> job state
|
||||
// GET ?action=memory_get the operator memory file and its budget
|
||||
// GET ?action=chats stored conversations, newest first, metadata only
|
||||
// GET ?action=chat_get&id=<hex32> one stored conversation with its transcript
|
||||
// GET ?action=findings [all=1] repair findings, open only unless all=1
|
||||
// POST action=finding_action id=<hex12> act=fix|move|ack|dismiss|reopen|cancel [note=…]
|
||||
// POST action=ask question=… [history=<JSON>] [kind=…] [think=0|1]
|
||||
// POST action=memory_set memory=… replace the memory file
|
||||
// POST action=clear token=<hex32> discard a finished job
|
||||
// POST action=chat_save [id=<hex32>] profile=… messages=<JSON>
|
||||
// POST action=chat_delete id=<hex32>
|
||||
//
|
||||
// RESPONSE
|
||||
// stats {"ok":true,"stats":{…}}
|
||||
// ask {"ok":true,"token":"<hex32>"}
|
||||
// poll {"ok":true,"job":{"status":"retrieving|generating|done|error",…}}
|
||||
// clear {"ok":true}
|
||||
// chats {"ok":true,"chats":[{id,ts,profile,title,turns}],"max":N}
|
||||
// findings {"ok":true,"repair":{enabled,autofix,last},"findings":[…],"counts":{…}}
|
||||
// finding_action {"ok":true,"action":"fix"}
|
||||
// chat_save {"ok":true,"id":"<hex32>","title":…}
|
||||
// {"ok":false,"error":…}
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/ai_actions.php vv_ai_dispatch() — every action, shared with the mesh entry point
|
||||
// include/ai_rpc.php vv_ai_route(), vv_ai_rpc() — where an action runs, and the SSH hop
|
||||
// include/ai.php vv_ai_enabled(), reached through ai_actions.php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// First executable statement, deliberately dependency-free. A request that is rejected by the
|
||||
// CSRF prepend never reaches here and a request that dies inside the include never reaches the
|
||||
// action log below, and those two look identical from outside — which is what made a POST that
|
||||
// the browser demonstrably sent leave no trace anywhere on the server.
|
||||
// Polls are excluded here for the same reason they are excluded from the action log below: one
|
||||
// line per second per open tab buries every line worth reading.
|
||||
if (strpos($_SERVER['REQUEST_URI'] ?? '', 'action=poll') === false) {
|
||||
@file_put_contents('/var/log/varaverk/ai.log',
|
||||
date('Y-m-d H:i:s') . ' ENTER ' . ($_SERVER['REQUEST_METHOD'] ?? '?')
|
||||
. ' ' . ($_SERVER['REQUEST_URI'] ?? '?')
|
||||
. ' ct=' . substr($_SERVER['CONTENT_TYPE'] ?? '-', 0, 40)
|
||||
. ' len=' . ($_SERVER['CONTENT_LENGTH'] ?? '-') . "\n", FILE_APPEND | LOCK_EX);
|
||||
}
|
||||
|
||||
|
||||
header('Content-Type: application/json');
|
||||
header('Cache-Control: no-store, no-cache');
|
||||
require_once dirname(__DIR__) . '/include/ai_actions.php';
|
||||
require_once dirname(__DIR__) . '/include/ai_rpc.php';
|
||||
|
||||
$isPost = $_SERVER['REQUEST_METHOD'] === 'POST';
|
||||
$action = trim($isPost ? ($_POST['action'] ?? '') : ($_GET['action'] ?? 'stats'));
|
||||
|
||||
// Params merged rather than picked by method. The POST-only checks inside the dispatcher are what
|
||||
// enforce the CSRF contract; which superglobal a value arrived in is not a security property, and
|
||||
// merging means a handler that reads one key does not care how the request was shaped.
|
||||
$params = $_POST + $_GET;
|
||||
|
||||
if ($action !== 'poll') {
|
||||
vv_ai_log(sprintf('%s action=%s from=%s',
|
||||
$_SERVER['REQUEST_METHOD'] ?? '?', $action ?: '(none)',
|
||||
$_SERVER['REMOTE_ADDR'] ?? '?'));
|
||||
}
|
||||
|
||||
// Master switch, ahead of everything. With AI_ENABLED false the tab is not in the tab list and no
|
||||
// dock is rendered, so nothing in the UI can legitimately reach any action here — including the
|
||||
// cheap reads, which would otherwise still answer with index and token figures for a subsystem the
|
||||
// operator has turned off. Not a 404: the switch is a setting, and the message names the setting.
|
||||
//
|
||||
// Checked before the routing below because it is this node's own switch either way. A mirror with
|
||||
// AI off must not forward to the owner: the operator turned AI off on this box, and honouring that
|
||||
// locally while quietly using someone else's model is not what the switch says it does.
|
||||
if (!vv_ai_enabled()) {
|
||||
echo json_encode(['ok' => false, 'error' => 'AI_ENABLED is false — AI features are off']);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Routing. vv_ai_route() decides local, remote or refused for this action on this node; the three
|
||||
// outcomes and the reasoning behind each live in include/ai_rpc.php, next to the transport that
|
||||
// carries them, rather than being restated here.
|
||||
$httpStatus = 200;
|
||||
$route = vv_ai_route($action);
|
||||
|
||||
if ($route === VV_AI_ROUTE_DENY) {
|
||||
http_response_code(404);
|
||||
echo json_encode(['ok' => false, 'error' => 'This AI surface lives on the owner node only']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$body = $route === VV_AI_ROUTE_REMOTE
|
||||
? vv_ai_rpc($action, $params, $isPost, $httpStatus)
|
||||
: vv_ai_dispatch($action, $params, $isPost, $httpStatus);
|
||||
|
||||
if ($httpStatus !== 200) http_response_code($httpStatus);
|
||||
echo json_encode($body);
|
||||
@@ -1,6 +1,80 @@
|
||||
<?php
|
||||
// Diagnostic endpoint — tests the Unraid GraphQL API and returns raw results.
|
||||
// Hit from browser: /plugins/varaverk/api/api_test.php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Unraid GraphQL API diagnostic. Reports whether this host has a usable API key, what the
|
||||
// PHP environment can do, and what the API actually answers — including live schema
|
||||
// introspection for the types the monitor layer depends on.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// A hand-run tool, not part of any page. Nothing in the UI links here; it is opened
|
||||
// directly at /plugins/varaverk/api/api_test.php when the monitor page starts showing
|
||||
// degraded detail and the question is whether the API, the key, or the parsing is at fault.
|
||||
//
|
||||
// It exists because that question was expensive to answer. The Unraid API renamed and
|
||||
// removed types between 7.2.5 and 7.3 — ArrayParity and ArrayCache stopped being distinct
|
||||
// types — and the only reliable way to know what the running version exposes is to ask it.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Reports the environment before the result.
|
||||
// Key presence, curl availability and allow_url_fopen are answered first, because a
|
||||
// failed probe means something entirely different depending on those three.
|
||||
//
|
||||
// Two transports, same query.
|
||||
// curl when available, a stream context otherwise. The fallback exists so the
|
||||
// diagnostic still returns something on a PHP build where the probe's failure would
|
||||
// otherwise be indistinguishable from the API being down.
|
||||
//
|
||||
// Returns raw alongside decoded.
|
||||
// probe_raw carries the first 1000 bytes verbatim. When the response is not JSON — an
|
||||
// HTML error page, a proxy interception — the decoded field is null and the raw text is
|
||||
// the only thing that explains why.
|
||||
//
|
||||
// Introspects the specific types the monitor layer reads.
|
||||
// Not a full schema dump. The named type lists are the ones whose field names the
|
||||
// collectors depend on, so a rename shows up here as a missing field rather than as a
|
||||
// quietly empty card on the monitor page.
|
||||
//
|
||||
// Pretty-printed on purpose. The only consumer is a person reading it in a browser tab.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Every query is a read or an introspection; nothing here mutates array, docker,
|
||||
// or VM state through the API.
|
||||
//
|
||||
// Every request is time-boxed.
|
||||
// 5s with a 3s connect timeout on the probe, 8s on each introspection call. A hung or
|
||||
// unreachable API returns a diagnostic rather than becoming one.
|
||||
//
|
||||
// The whole probe is skipped without a key, and reports that as the finding.
|
||||
// key_present is false and probe stays null. The absence of a key is the most common
|
||||
// answer this tool gives, and it is reported as a result rather than as a failure.
|
||||
//
|
||||
// Localhost only. The endpoint is hardcoded to http://localhost/graphql — no part of the
|
||||
// request selects a target, so this cannot be used to probe another host.
|
||||
//
|
||||
// The key is truncated in the response.
|
||||
// Only the first 8 characters are returned, as key_prefix — enough to confirm which key
|
||||
// is in use, not enough to use it.
|
||||
//
|
||||
// Known exposure: this returns more than a normal endpoint should.
|
||||
// A key prefix, the cached API debug log, and the live schema are all visible to anyone
|
||||
// with a WebGUI session. That is acceptable for a diagnostic reachable only by typing
|
||||
// its URL, but it is the reason it is not linked from any page and should not be
|
||||
// wrapped in one. It is a pure GET read, so Unraid's POST-only CSRF guard does not
|
||||
// apply to it — the WebGUI session is its whole boundary. See README-unraid.md.
|
||||
//
|
||||
// REQUEST
|
||||
// GET, no parameters
|
||||
//
|
||||
// RESPONSE
|
||||
// {"host","key_present","key_prefix","curl_available","allow_url_fopen","debug_log",
|
||||
// "probe":{"url","http_code","curl_err","decoded"},"probe_raw",
|
||||
// "schema","pool_drives","schema2"}
|
||||
// The schema, pool_drives and schema2 keys are present only when a key exists.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/config.php vv_detect_host(), vv_conf_vars(), VV_CACHE_DIR
|
||||
// Unraid API http://localhost/graphql
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
|
||||
@@ -1,11 +1,87 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Arr data endpoint. Serves the arrs tab: local Sonarr/Radarr/Lidarr statistics gathered
|
||||
// live, partner statistics served from the background cache, plus an on-demand refresh of
|
||||
// one partner's cache.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Two modes on one URL. Without parameters it answers from a 300s cache and only builds a
|
||||
// fresh payload on a miss. With ?action=refresh_remote&host=hostN it shells out to
|
||||
// Tools/remote_arr_cache_writer.sh for that one host, returns the newly written node, and
|
||||
// busts the main cache so the next ordinary poll picks the change up.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// The refresh branch runs before the cache read, and exits.
|
||||
// It is a distinct operation, not a cache-control flag, so it never falls through into
|
||||
// the normal load path and cannot return a stale payload labelled as refreshed.
|
||||
//
|
||||
// Cache first for the ordinary case.
|
||||
// vv_arrs_all() contacts every configured arr instance. At the tab's poll rate that is
|
||||
// far too expensive to repeat, so the 300s cache is the default path and the live build
|
||||
// is the exception.
|
||||
//
|
||||
// include/arrs.php is required only on a cache miss.
|
||||
// A cache hit answers without loading the library at all, which is the difference
|
||||
// between a poll that costs a file read and one that costs an autoload.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// The host parameter is matched against a pattern, never used as a path.
|
||||
// ^host\d+$ is enforced before the value goes anywhere. It reaches the script only as
|
||||
// an escapeshellarg'd --host= value and the cache filename it composes, so neither a
|
||||
// shell metacharacter nor a traversal sequence can survive the check.
|
||||
//
|
||||
// The refresh is externally time-boxed.
|
||||
// set_time_limit() does not count time spent inside exec() on Linux, so PHP's own limit
|
||||
// cannot end a hung SSH call — the child is wrapped in `timeout` instead. Exit 124 is
|
||||
// reported as a timeout rather than a generic failure, so the UI can distinguish an
|
||||
// unreachable partner from a broken script.
|
||||
//
|
||||
// A missing script is reported, not executed.
|
||||
// The file_exists() check runs before exec(), so a partial deploy returns a named error
|
||||
// rather than a shell "command not found" surfacing as an empty refresh.
|
||||
//
|
||||
// The cache is busted after the write, not before.
|
||||
// @unlink() of arrs.json follows the script run, so a failed refresh leaves the previous
|
||||
// good payload in place instead of forcing every subsequent poll onto the live path.
|
||||
//
|
||||
// Read-only with respect to the arrs themselves. Statistics come from the databases the
|
||||
// scripts write; nothing here triggers a scan, cleanup, or import.
|
||||
//
|
||||
// REQUEST
|
||||
// GET cached (300s) full payload
|
||||
// POST action=refresh_remote host=host<n> re-run the partner cache writer for one host
|
||||
// (POST, so Unraid's CSRF guard applies)
|
||||
//
|
||||
// RESPONSE
|
||||
// normal vv_arrs_all() verbatim — local node live, remote nodes cached with cache_age
|
||||
// refresh {"ok":bool,"node":object|null,"output":string}
|
||||
// errors {"ok":false,"error":string} — invalid host, missing script, or timeout
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/config.php vv_cache_read(), VV_CACHE_DIR
|
||||
// include/arrs.php vv_arrs_all() (loaded only on a cache miss)
|
||||
// Tools/remote_arr_cache_writer.sh (refresh branch only)
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
// Bound on the partner refresh child process. set_time_limit() does not cover exec() time
|
||||
// on Linux, so this is enforced by `timeout`, not by PHP.
|
||||
define('VV_ARRS_REFRESH_TIMEOUT', 120);
|
||||
|
||||
// ── Manual remote refresh — runs remote_arr_cache_writer for one host ─────────
|
||||
$_action = trim($_GET['action'] ?? '');
|
||||
// POST only. The refresh executes a script, and Unraid's CSRF prepend validates POSTs while
|
||||
// ignoring GETs entirely — so reaching this over GET would mean running it with no token
|
||||
// check. A GET naming the action is refused rather than falling through to the cached read,
|
||||
// so a stale caller fails visibly instead of silently appearing to succeed.
|
||||
if (trim($_GET['action'] ?? '') === 'refresh_remote') {
|
||||
http_response_code(405);
|
||||
echo json_encode(['ok' => false, 'error' => 'POST only']); exit;
|
||||
}
|
||||
$_action = ($_SERVER['REQUEST_METHOD'] === 'POST') ? trim($_POST['action'] ?? '') : '';
|
||||
if ($_action === 'refresh_remote') {
|
||||
$host = strtolower(trim($_GET['host'] ?? ''));
|
||||
$host = strtolower(trim($_POST['host'] ?? ''));
|
||||
if (!preg_match('/^host\d+$/', $host)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid host']); exit;
|
||||
}
|
||||
@@ -13,9 +89,19 @@ if ($_action === 'refresh_remote') {
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'remote_arr_cache_writer.sh not found']); exit;
|
||||
}
|
||||
set_time_limit(30);
|
||||
set_time_limit(VV_ARRS_REFRESH_TIMEOUT + 30);
|
||||
$out = []; $exit = 0;
|
||||
exec('bash ' . escapeshellarg($script) . ' --host=' . escapeshellarg(strtoupper($host)) . ' 2>&1', $out, $exit);
|
||||
exec('timeout ' . VV_ARRS_REFRESH_TIMEOUT . ' bash ' . escapeshellarg($script)
|
||||
. ' --host=' . escapeshellarg(strtoupper($host)) . ' 2>&1', $out, $exit);
|
||||
|
||||
if ($exit === 124) {
|
||||
echo json_encode([
|
||||
'ok' => false,
|
||||
'error' => 'Refresh timed out after ' . VV_ARRS_REFRESH_TIMEOUT . 's',
|
||||
'output' => implode("\n", $out),
|
||||
]);
|
||||
exit;
|
||||
}
|
||||
|
||||
$cacheFile = VV_CACHE_DIR . '/arrs_remote_' . $host . '.json';
|
||||
$node = null;
|
||||
|
||||
+262
-3
@@ -1,9 +1,257 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Auth stack endpoint. The single URL behind the auth tab, covering all three services it
|
||||
// manages: Nginx Proxy Manager proxy hosts and certificates, lldap users and groups, and
|
||||
// Authelia access rules.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Read and write share one URL, split on HTTP method. GET serves the five read actions and
|
||||
// is always safe. POST carries an action naming exactly one library call. Anything that is
|
||||
// neither GET nor POST is refused with 405 before a parameter is read.
|
||||
//
|
||||
// Three services, one endpoint, because they are one subject. The auth stack is HOST1's
|
||||
// source of truth for identity, and a change in one service usually accompanies a change in
|
||||
// another — a new lldap user is only useful once an Authelia rule and an NPM host exist for
|
||||
// it. Splitting them into three endpoints would split one page's work across three files
|
||||
// with three copies of the same dispatch.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Both dispatches are closed match expressions.
|
||||
// Every action maps to one named library call, and an unrecognised action falls to a
|
||||
// default arm that returns an error. No part of the request is ever used to construct a
|
||||
// function name, so the action list is the complete set of things this endpoint can do.
|
||||
//
|
||||
// The endpoint holds no auth logic of its own.
|
||||
// Token acquisition, API dialects, config parsing and the Authelia container restart
|
||||
// all live in include/auth.php. This file is dispatch and nothing else, which is what
|
||||
// keeps the credential handling in one auditable place.
|
||||
//
|
||||
// Structured payloads arrive as JSON in a form field.
|
||||
// Proxy definitions and rule sets are nested, so they are passed as encoded JSON rather
|
||||
// than flattened into form keys — a rule set does not survive form encoding intact.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Wrong method is refused with a status code, not just a body, so a mistaken caller fails
|
||||
// visibly rather than parsing an error object as data.
|
||||
//
|
||||
// Every parameter is optional and typed at the call site.
|
||||
// (int) casts on ids, ?? '' on strings, ?? '0' === '1' on flags. A malformed POST
|
||||
// reaches the library as zeros and empty strings — which the library rejects — rather
|
||||
// than raising undefined-index warnings into the JSON body and corrupting the response.
|
||||
//
|
||||
// Malformed JSON degrades to an empty structure.
|
||||
// json_decode with ?: [] on both data and rules. A truncated payload becomes an empty
|
||||
// set the library refuses, not a partial one it might act on.
|
||||
//
|
||||
// Credentials pass through, and are never returned.
|
||||
// lldap_set_password and lldap_create_user accept a password and hand it straight to
|
||||
// the library. No action in either dispatch returns a stored credential, and nothing
|
||||
// here writes one to a log.
|
||||
//
|
||||
// The destructive actions are POST-only by construction.
|
||||
// Delete of a proxy, a user, or a group exists only in the POST match. The GET arm has
|
||||
// five read actions and no others, so no link or prefetch can reach a delete.
|
||||
//
|
||||
// Authelia rule writes are atomic and refuse to create.
|
||||
// The library writes .vv.tmp and renames, and returns 'Config not found' rather than
|
||||
// authoring a fresh config — a config it created would carry no rules and a default
|
||||
// policy, which is an accidental open door.
|
||||
//
|
||||
// The POST actions are CSRF-guarded by the platform, not by this file.
|
||||
// Unraid's auto_prepend (webGui/local_prepend.php) validates a token on every POST and
|
||||
// terminates the request before any code here runs. That guard is the reason every
|
||||
// mutating action lives in the POST arm — this is the highest-value endpoint in the
|
||||
// plugin to reach, since it can create a user and open a proxy host. The five GET
|
||||
// actions are reads and are deliberately outside it. See README-unraid.md.
|
||||
//
|
||||
// REQUEST
|
||||
// GET ?action=npm_proxies | npm_certs | npm_stats | npm_uptime | lldap_users | lldap_groups
|
||||
// | authelia_rules
|
||||
// GET ?action=npm_why id
|
||||
// GET ?action=access_check domain, uid, path
|
||||
// POST action=npm_create data=<JSON>
|
||||
// POST action=npm_update id, data=<JSON>
|
||||
// POST action=npm_delete id
|
||||
// POST action=npm_toggle id, enabled=0|1
|
||||
// POST action=lldap_create_user uid, email, display_name, password
|
||||
// POST action=lldap_update_user uid, email, display_name
|
||||
// POST action=lldap_delete_user uid
|
||||
// POST action=lldap_set_password uid, password
|
||||
// POST action=lldap_create_group name
|
||||
// POST action=lldap_delete_group id
|
||||
// POST action=lldap_add_to_group uid, gid
|
||||
// POST action=lldap_remove_from_group uid, gid
|
||||
// POST action=authelia_save rules=<JSON>, default_policy
|
||||
//
|
||||
// RESPONSE
|
||||
// Whatever the invoked library call returns — ['ok' => bool] with a payload or an error,
|
||||
// or an _err key on a failed remote call. npm_certs is wrapped as {"ok":true,"certs":[…]}.
|
||||
// {"ok":false,"error":"Unknown action: …"} for anything outside the lists above.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/auth.php vv_npm_*(), vv_lldap_*(), vv_authelia_read_rules(),
|
||||
// vv_authelia_write_rules()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
|
||||
// Which panel each action belongs to. AUTH_STACK decides which panels the tab draws, and this is
|
||||
// the same decision applied to the endpoint — a tab left open from before a switch would otherwise
|
||||
// keep writing to the stack that is no longer in force, which on this page means editing the
|
||||
// directory or the rules of a system nobody is authenticating against any more.
|
||||
//
|
||||
// Proxies and certs are Nginx Proxy Manager's, not the identity stack's, so they are listed under
|
||||
// panels every stack carries rather than gated to one.
|
||||
const VV_AUTH_ACTION_PANEL = [
|
||||
// GET
|
||||
'npm_proxies' => 'proxies', 'npm_certs' => 'proxies', 'npm_stats' => 'proxies', 'npm_uptime' => 'proxies',
|
||||
'npm_why' => 'proxies',
|
||||
'lldap_users' => 'users', 'lldap_groups' => 'users', 'lldap_avatar' => 'users',
|
||||
'authelia_rules' => 'acl', 'access_check' => 'acl',
|
||||
// POST
|
||||
'npm_create' => 'proxies', 'npm_update' => 'proxies',
|
||||
'npm_delete' => 'proxies', 'npm_toggle' => 'proxies',
|
||||
'lldap_create_user' => 'users', 'lldap_update_user' => 'users', 'lldap_delete_user' => 'users',
|
||||
'lldap_set_password' => 'users', 'lldap_set_avatar' => 'users', 'lldap_remove_avatar' => 'users',
|
||||
'lldap_create_group' => 'users', 'lldap_delete_group' => 'users', 'lldap_rename_group' => 'users',
|
||||
'lldap_add_to_group' => 'users', 'lldap_remove_from_group' => 'users',
|
||||
'authelia_save' => 'acl',
|
||||
];
|
||||
|
||||
// The windowing rule lives in include/auth.php and is shared with Tools/uptime_probe.php. It was
|
||||
// three separate copies of the same six lines, which is three places for the definition of "the
|
||||
// last 24 hours" to drift apart while every one of them keeps returning a plausible number.
|
||||
function vv_uptime_window_api(array $buckets, int $n): ?float {
|
||||
return vv_auth_uptime_window($buckets, $n);
|
||||
}
|
||||
|
||||
// One period of the history card: the rolled-up percentage, the drawable series, and how much of
|
||||
// the window has actually been observed. All three come off the same buckets, so the number and
|
||||
// the graph beside it can never disagree.
|
||||
function vv_uptime_period(array $buckets, int $n, string $unit): array {
|
||||
$series = vv_auth_uptime_series($buckets, $n, $unit);
|
||||
return [
|
||||
'pct' => vv_auth_uptime_window($buckets, $n),
|
||||
'series' => $series,
|
||||
'have' => count(array_filter($series, fn($v) => $v !== null)),
|
||||
'want' => $n,
|
||||
];
|
||||
}
|
||||
|
||||
function vv_auth_action_allowed(string $action): bool {
|
||||
$panel = VV_AUTH_ACTION_PANEL[$action] ?? null;
|
||||
// Unmapped actions are left to the existing "Unknown action" answer rather than being refused
|
||||
// here, so a new action is never silently blocked by a table someone forgot to extend.
|
||||
return $panel === null || vv_auth_panel_on($panel);
|
||||
}
|
||||
|
||||
function vv_auth_action_refusal(string $action): array {
|
||||
$d = vv_auth_stack_def();
|
||||
return ['ok' => false, 'error' => 'AUTH_STACK is "' . vv_auth_stack() . '" (' . $d['label']
|
||||
. '), which does not serve this request. Reload the Auth tab.'];
|
||||
}
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] === 'GET') {
|
||||
$action = $_GET['action'] ?? '';
|
||||
if (!vv_auth_action_allowed($action)) { echo json_encode(vv_auth_action_refusal($action)); exit; }
|
||||
|
||||
// The one route here that does not answer in JSON — it streams the stored JPEG so the page can
|
||||
// point an <img> at it, rather than carrying 470 KB of base64 through the user list on every
|
||||
// load. Handled before the match so the Content-Type set above is replaced rather than sent
|
||||
// alongside image bytes.
|
||||
if ($action === 'lldap_avatar') {
|
||||
$raw = vv_lldap_avatar((string) ($_GET['uid'] ?? ''));
|
||||
if ($raw === '') { header('Content-Type: application/json'); http_response_code(404);
|
||||
echo json_encode(['ok' => false, 'error' => 'No avatar']); exit; }
|
||||
header('Content-Type: image/jpeg');
|
||||
header('Content-Length: ' . strlen($raw));
|
||||
// Private, because this is a photograph of a person behind an authenticated admin page,
|
||||
// and must not be held by anything between here and the browser. Short, because the
|
||||
// operator changing an avatar expects to see it change.
|
||||
header('Cache-Control: private, max-age=60');
|
||||
echo $raw;
|
||||
exit;
|
||||
}
|
||||
|
||||
// Per-host request and byte totals, written by Tools/npm_access_stats.sh. Served rather than
|
||||
// computed: the logs behind these numbers are 475 MB and reading them is a scheduled job, not
|
||||
// something a page load can do.
|
||||
if ($action === 'npm_stats') {
|
||||
$f = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/npm_access.json';
|
||||
$s = is_file($f) ? (json_decode((string) @file_get_contents($f), true) ?: []) : [];
|
||||
echo json_encode(['ok' => true, 'hosts' => $s['hosts'] ?? [],
|
||||
'last_pass' => $s['last_pass'] ?? null]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Per-domain uptime, written by Tools/uptime_probe.sh every minute. Keyed by hostname rather
|
||||
// than proxy id, because that is what was probed.
|
||||
if ($action === 'npm_uptime') {
|
||||
$f = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/uptime.json';
|
||||
$u = is_file($f) ? (json_decode((string) @file_get_contents($f), true) ?: []) : [];
|
||||
$out = [];
|
||||
foreach ($u['domains'] ?? [] as $dom => $r) {
|
||||
// Only what the row draws. The hourly and daily buckets are dozens of entries per
|
||||
// domain and the page shows three percentages and a strip.
|
||||
$out[$dom] = [
|
||||
'state' => $r['state'] ?? null,
|
||||
'samples' => array_slice($r['samples'] ?? [], -60),
|
||||
'h1' => vv_uptime_window_api($r['hours'] ?? [], 1),
|
||||
'h24' => vv_uptime_window_api($r['hours'] ?? [], 24),
|
||||
'd30' => vv_uptime_window_api($r['days'] ?? [], 30),
|
||||
'last_change' => $r['last_change'] ?? null,
|
||||
'last_detail' => $r['last_detail'] ?? null,
|
||||
'last_ms' => $r['last_ms'] ?? null,
|
||||
// When this domain was last actually probed. The probe follows NPM's host list, so
|
||||
// a domain removed there simply stops being visited while its record stays here —
|
||||
// and a stored state of "down" then describes the moment it left, not today. The
|
||||
// card needs this to tell "broken" from "no longer served".
|
||||
'last_at' => $r['last_at'] ?? null,
|
||||
// The history card below the table. One aligned series per period rather than the
|
||||
// raw buckets: the client would otherwise have to re-derive calendar keys to know
|
||||
// which of thirty slots a given day belongs in, and there would then be two
|
||||
// implementations of that rule in two languages.
|
||||
//
|
||||
// `have` is what actually exists, so the card can say "collecting — 2 of 30 days"
|
||||
// instead of printing a percentage computed from two days as though it were a
|
||||
// month. The store began 2026-08-15; every window longer than a day is partial
|
||||
// for a while, and a confident figure over a short sample is the one thing this
|
||||
// card must not do.
|
||||
'hist' => [
|
||||
'h24' => vv_uptime_period($r['hours'] ?? [], 24, 'hour'),
|
||||
'd7' => vv_uptime_period($r['days'] ?? [], 7, 'day'),
|
||||
'd30' => vv_uptime_period($r['days'] ?? [], 30, 'day'),
|
||||
'm12' => vv_uptime_period($r['months'] ?? [], 12, 'month'),
|
||||
],
|
||||
];
|
||||
}
|
||||
echo json_encode(['ok' => true, 'domains' => $out, 'last_pass' => $u['last_pass'] ?? null]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Why one host is not at 100%. The only read here that goes and looks rather than serving a
|
||||
// stored figure: it opens a socket to the forward target, asks the domain itself, and inspects
|
||||
// the containers. Slow by the standards of this file — several seconds — which is why it is one
|
||||
// host on demand and never part of the list load.
|
||||
//
|
||||
// A read, so it stays in the GET arm with the other reads. It is worth being explicit that this
|
||||
// is safe to leave outside the CSRF guard: every call it makes is a GET, a HEAD, a TCP connect
|
||||
// or a file read, so the worst a forged request achieves is making this machine look at itself.
|
||||
if ($action === 'npm_why') {
|
||||
echo json_encode(vv_npm_why((int) ($_GET['id'] ?? 0)));
|
||||
exit;
|
||||
}
|
||||
|
||||
// Can this user open this URL, and what decided it. Reads NPM, the Authelia instance that this
|
||||
// particular host talks to, and the directory — the three places the answer is split across.
|
||||
if ($action === 'access_check') {
|
||||
echo json_encode(vv_auth_access_check((string) ($_GET['domain'] ?? ''),
|
||||
(string) ($_GET['uid'] ?? ''),
|
||||
(string) ($_GET['path'] ?? '/')));
|
||||
exit;
|
||||
}
|
||||
|
||||
$result = match ($action) {
|
||||
'npm_proxies' => vv_npm_list_proxies(),
|
||||
'npm_certs' => ['ok' => true, 'certs' => vv_npm_list_certs()],
|
||||
@@ -23,6 +271,7 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
}
|
||||
|
||||
$action = trim($_POST['action'] ?? '');
|
||||
if (!vv_auth_action_allowed($action)) { echo json_encode(vv_auth_action_refusal($action)); exit; }
|
||||
|
||||
$result = match ($action) {
|
||||
// NPM
|
||||
@@ -31,8 +280,16 @@ $result = match ($action) {
|
||||
'npm_delete' => vv_npm_delete_proxy((int)($_POST['id'] ?? 0)),
|
||||
'npm_toggle' => vv_npm_toggle_proxy((int)($_POST['id'] ?? 0), ($_POST['enabled'] ?? '0') === '1'),
|
||||
// lldap
|
||||
'lldap_create_user' => vv_lldap_create_user($_POST['uid'] ?? '', $_POST['email'] ?? '', $_POST['display_name'] ?? '', $_POST['password'] ?? ''),
|
||||
'lldap_update_user' => vv_lldap_update_user($_POST['uid'] ?? '', $_POST['email'] ?? '', $_POST['display_name'] ?? ''),
|
||||
'lldap_create_user' => vv_lldap_create_user($_POST['uid'] ?? '', $_POST['email'] ?? '', $_POST['display_name'] ?? '', $_POST['password'] ?? '',
|
||||
$_POST['first_name'] ?? '', $_POST['last_name'] ?? ''),
|
||||
// isset, not ??'' — the update helper reads null as "not offered" and '' as "cleared", and
|
||||
// collapsing the two here would erase a first name every time a form omitted the field.
|
||||
'lldap_update_user' => vv_lldap_update_user($_POST['uid'] ?? '', $_POST['email'] ?? '', $_POST['display_name'] ?? '',
|
||||
isset($_POST['first_name']) ? (string) $_POST['first_name'] : null,
|
||||
isset($_POST['last_name']) ? (string) $_POST['last_name'] : null),
|
||||
'lldap_set_avatar' => vv_lldap_set_avatar($_POST['uid'] ?? '', $_POST['avatar'] ?? ''),
|
||||
'lldap_remove_avatar' => vv_lldap_remove_avatar($_POST['uid'] ?? ''),
|
||||
'lldap_rename_group' => vv_lldap_rename_group((int)($_POST['id'] ?? 0), $_POST['name'] ?? ''),
|
||||
'lldap_delete_user' => vv_lldap_delete_user($_POST['uid'] ?? ''),
|
||||
'lldap_set_password' => vv_lldap_set_password($_POST['uid'] ?? '', $_POST['password'] ?? ''),
|
||||
'lldap_create_group' => vv_lldap_create_group($_POST['name'] ?? ''),
|
||||
@@ -40,7 +297,9 @@ $result = match ($action) {
|
||||
'lldap_add_to_group' => vv_lldap_add_to_group($_POST['uid'] ?? '', (int)($_POST['gid'] ?? 0)),
|
||||
'lldap_remove_from_group' => vv_lldap_remove_from_group($_POST['uid'] ?? '', (int)($_POST['gid'] ?? 0)),
|
||||
// Authelia
|
||||
'authelia_save' => vv_authelia_write_rules(json_decode($_POST['rules'] ?? '[]', true) ?: [], $_POST['default_policy'] ?? 'deny'),
|
||||
'authelia_save' => vv_authelia_write_rules(json_decode($_POST['rules'] ?? '[]', true) ?: [],
|
||||
$_POST['default_policy'] ?? 'deny',
|
||||
$_POST['default_note'] ?? ''),
|
||||
default => ['ok' => false, 'error' => 'Unknown action: ' . $action],
|
||||
};
|
||||
|
||||
|
||||
@@ -1,5 +1,101 @@
|
||||
<?php
|
||||
// Live board data: locks, recent errors, partner reachability.
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Attention board. The three things worth knowing without opening a tab: locks nobody is
|
||||
// holding, scripts whose last run ended badly, and whether the partner is reachable.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// A dashboard of exceptions, not of state. Everything here is designed to be empty on a
|
||||
// healthy host — an empty board is the expected reading, which is what makes a non-empty
|
||||
// one worth looking at.
|
||||
//
|
||||
// Each section answers a question no single other endpoint does. Locks come from the
|
||||
// filesystem, errors from correlating logs against their stat files, reachability from an
|
||||
// actual ping. They are collected together because they are read together.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Only stale locks are reported.
|
||||
// A lock whose owning pid is still in /proc is a job legitimately running and is
|
||||
// skipped. What remains is the set a person might need to clear — which is exactly what
|
||||
// clearlock.php exists for.
|
||||
//
|
||||
// Errors are gated on the stat file, not on the log text.
|
||||
// A script is only reported when its last recorded run exited warn or error. Scanning
|
||||
// logs for the word "error" produced two persistent false positives: dry runs, which
|
||||
// print failures they did not cause, and success summaries containing lines like
|
||||
// "Failed: 0". The stat file is the run's own verdict, and it is authoritative.
|
||||
//
|
||||
// A script with no stat file is skipped entirely.
|
||||
// No stat file means it never ran through run_job.sh, so there is no verdict to trust
|
||||
// and no basis for reporting it.
|
||||
//
|
||||
// The error line is the last matching one, searched backwards.
|
||||
// Logs are appended, so the most recent failure is at the end. The search walks up from
|
||||
// there and stops at the first hit rather than reading forward and reporting the oldest.
|
||||
//
|
||||
// There is a fallback when nothing matches.
|
||||
// A run that failed without printing a recognisable error still reports its last
|
||||
// non-blank line. A script marked error with no explanation is worse than an imperfect
|
||||
// one.
|
||||
//
|
||||
// The partner is discovered, not configured.
|
||||
// master.conf is scanned for HOST<n> entries and the first non-self one is used, so this
|
||||
// works for any number of hosts without a second list to maintain.
|
||||
//
|
||||
// Tailscale resolves the address, never local DNS.
|
||||
// vv_resolve_tailscale_ip() mirrors common.sh, so the board tests the same path the
|
||||
// rest of the system uses and survives the partner's IP changing.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. Nothing here clears a lock, truncates a log, or restarts anything — the board
|
||||
// reports; clearlock.php and stop.php act.
|
||||
//
|
||||
// The lock scan is bounded to a hardcoded directory.
|
||||
// glob over /tmp/unraid_locks/*.lock — a literal, not a config value, so no conf edit
|
||||
// can point this scan somewhere else.
|
||||
//
|
||||
// The log walk is wrapped in a try/catch.
|
||||
// RecursiveDirectoryIterator throws when LOG_DIR is absent or a subdirectory is
|
||||
// unreadable — the normal state on a fresh install. The catch yields an empty error
|
||||
// list rather than a 500.
|
||||
//
|
||||
// Every file read is independently suppressed and defaulted.
|
||||
// @file_get_contents, @file, and ?: fallbacks throughout. One unreadable log or stat
|
||||
// file costs its own row, not the response.
|
||||
//
|
||||
// The scan window and the output are both bounded.
|
||||
// Logs older than 7 days are skipped, only the last 200 lines of each are read, each
|
||||
// line is truncated at 220 characters, and the result is capped at 20 errors. This runs
|
||||
// against a directory that grows without limit.
|
||||
//
|
||||
// ANSI escapes are stripped before matching and before returning.
|
||||
// Logs are written with colour. Without stripping, the patterns would miss coloured
|
||||
// error markers and the JSON would carry terminal control codes into the page.
|
||||
//
|
||||
// The ping target is escaped, and time-boxed by ping itself.
|
||||
// escapeshellarg() on a value that came from conf, and -c1 -W2 so an unreachable
|
||||
// partner costs two seconds. The result is cached 30s so the board's poll does not ping
|
||||
// on every request.
|
||||
//
|
||||
// Reachability falls back to the hostname when Tailscale cannot resolve.
|
||||
// Better to test something and report the result than to report nothing because the
|
||||
// preferred resolution path failed.
|
||||
//
|
||||
// REQUEST
|
||||
// GET, no parameters
|
||||
//
|
||||
// RESPONSE
|
||||
// {"ok":true,
|
||||
// "locks":[{"name","file","age"}], stale locks only — empty is healthy
|
||||
// "errors":[{"script","line","ts"}], newest first, max 20
|
||||
// "partner":{"host","reachable","latency"}} null when no partner is configured
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/config.php LOG_DIR, vv_conf_vars(), vv_get_hostname(),
|
||||
// vv_resolve_tailscale_ip(), vv_cache_read(), vv_cache_write()
|
||||
// /tmp/unraid_locks lock files written by common.sh's locking helper
|
||||
// LOG_DIR/** .log files and their .json stat files, written by run_job.sh
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
@@ -74,7 +170,7 @@ if (is_dir(LOG_DIR)) {
|
||||
if ($lastErr !== null)
|
||||
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()];
|
||||
}
|
||||
} catch (Exception $e) {}
|
||||
} catch (Exception $e) { vv_log_error('api/board.php', 'log walk failed: ' . $e->getMessage()); }
|
||||
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
||||
}
|
||||
$out['errors'] = array_slice($errors, 0, 20);
|
||||
|
||||
+194
-2
@@ -1,4 +1,97 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Certificate status. Serves the cert panel four ways — the cached results of the last
|
||||
// cert_monitor.sh run, the configured domain list, a live read of NPM's own certificate
|
||||
// store, and an on-demand re-run of the monitor.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Two independent sources of truth, deliberately kept separate. cert_monitor.sh checks the
|
||||
// domains as they resolve from outside — the certificate a visitor actually receives. NPM's
|
||||
// API reports what it holds internally. They disagree exactly when something is wrong: a
|
||||
// renewed cert that was never reloaded, or a proxy host pointing at the wrong certificate.
|
||||
// Collapsing them into one number would hide the only case worth catching.
|
||||
//
|
||||
// The default path serves cert_monitor.sh's cache and never checks anything itself. That
|
||||
// script runs on the Sunday report schedule; the run action exists for when someone does
|
||||
// not want to wait for it.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Thresholds come from master.conf, with shipped defaults.
|
||||
// CERT_WARN_DAYS and CERT_CRIT_DAYS are read per request rather than baked in, so the
|
||||
// page and the script that notifies agree on what "critical" means.
|
||||
//
|
||||
// No cache is answered with the domain list, not with an error.
|
||||
// Before the first run there is nothing to report, so the configured domains are
|
||||
// returned marked UNKN. The panel shows what is being watched rather than an empty
|
||||
// state that looks like nothing is configured.
|
||||
//
|
||||
// Already expired sorts with critical, not past it.
|
||||
// A negative day count is CRIT rather than a separate state, and the sort puts the
|
||||
// smallest number first — so the most urgent certificate is always at the top.
|
||||
//
|
||||
// The run action returns the script's output alongside the fresh data.
|
||||
// Capped at 30 lines. When a check fails, the reason is in that output and nowhere in
|
||||
// the structured result.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// The action is chosen from a fixed set, and everything else falls to the cached read.
|
||||
// No part of the request names a script, a domain, or a file. The one script this
|
||||
// endpoint can run is a hardcoded path.
|
||||
//
|
||||
// The script run is externally time-boxed.
|
||||
// `timeout 180` wraps it — set_time_limit() does not cover exec() time on Linux, so
|
||||
// PHP's own limit cannot end a check hung on an unresponsive domain. Exit 124 is
|
||||
// reported as a timeout rather than a generic failure.
|
||||
//
|
||||
// A missing script is reported, not executed.
|
||||
// file_exists() precedes exec(), so a partial deploy returns a named error rather than
|
||||
// a shell failure surfacing as an empty result.
|
||||
//
|
||||
// Every threshold read has a default.
|
||||
// ?: 30 and ?: 7 on the regex captures, so a master.conf mid-edit or missing the keys
|
||||
// still yields coherent statuses instead of comparing every certificate against zero
|
||||
// and reporting the whole estate critical.
|
||||
//
|
||||
// Missing or malformed expiry data is UNKN, never OK.
|
||||
// A certificate whose expires_on is absent or unparseable yields a null day count and
|
||||
// an explicit unknown status. Defaulting it to OK would silently drop a certificate out
|
||||
// of monitoring — the one failure this panel exists to prevent.
|
||||
//
|
||||
// Every cache read degrades.
|
||||
// json_decode with ?: fallbacks throughout, so a truncated cache file yields an empty
|
||||
// result rather than a fatal.
|
||||
//
|
||||
// The NPM path reports its own transport failures.
|
||||
// vv_npm_req() returns an _err key rather than throwing, and that is checked before the
|
||||
// response is treated as a certificate list — so an unreachable NPM is reported as such
|
||||
// instead of rendering as zero certificates.
|
||||
//
|
||||
// The run action is POST only, which is what places it behind Unraid's CSRF guard.
|
||||
// The platform prepend validates the token on every POST and inspects no GET at all, so
|
||||
// an action that executes a script must not be reachable by GET. The three read actions
|
||||
// stay GET-reachable because they change nothing.
|
||||
//
|
||||
// REQUEST
|
||||
// GET cached status, or the configured domains when no cache exists
|
||||
// GET|POST ?action=npm live certificate list from NPM's API
|
||||
// GET|POST ?action=domains configured domains and thresholds, no checks run
|
||||
// POST action=run re-run cert_monitor.sh, then return its fresh cache
|
||||
// (POST, so Unraid's CSRF guard applies)
|
||||
//
|
||||
// RESPONSE
|
||||
// default {"ok":true,"checked_at","host","warn_days","crit_days","domains":[…]}
|
||||
// npm {"ok":true,"certs":[{id,nice_name,domain_names,provider,expires,days,status}],
|
||||
// "warn_days","crit_days"}
|
||||
// domains {"ok":true,"host_id","domains":[…],"warn_days","crit_days"}
|
||||
// run {"ok":true,"data":…,"output":[…],"rc":int}
|
||||
// {"ok":false,"error":"NPM request failed"|"cert_monitor.sh not found"|"… timed out …"}
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/config.php vv_detect_host(), vv_read_conf_raw(), STATE_DIR, SCRIPTS_DIR
|
||||
// include/auth.php vv_npm_req()
|
||||
// Monitors/cert_monitor.sh writes STATE_DIR/cert_status.json
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
require_once dirname(__DIR__) . '/include/auth.php';
|
||||
@@ -46,6 +139,82 @@ if ($action === 'npm') {
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Per-domain history, totals, and the DDNS containers ──────────────────────
|
||||
// Read-only. Tools/cert_history.sh is what writes the store; serving it from here would mean the
|
||||
// counters only advance when somebody happens to have the tab open.
|
||||
if ($action === 'history') {
|
||||
$file = rtrim(defined('DB_DIR') ? DB_DIR : (DATA_DIR . '/db'), '/') . '/cert_history.json';
|
||||
$hist = is_file($file) ? (json_decode((string) @file_get_contents($file), true) ?: []) : [];
|
||||
$doms = is_array($hist['domains'] ?? null) ? $hist['domains'] : [];
|
||||
|
||||
$now = time();
|
||||
$rows = [];
|
||||
$tot = ['tracked' => 0, 'active' => 0, 'retired' => 0, 'removed' => 0,
|
||||
'renewals' => 0, 'failures' => 0, 'checks' => 0, 'oldest' => null];
|
||||
|
||||
foreach ($doms as $domain => $r) {
|
||||
$first = (int) ($r['first_seen'] ?? $now);
|
||||
$retired = !empty($r['retired_at']);
|
||||
$removed = !empty($r['removed_at']);
|
||||
$rows[] = [
|
||||
'domain' => (string) $domain,
|
||||
'first_seen'=> $first,
|
||||
'tracked' => vv_cert_span_php($first, $now),
|
||||
'checks' => (int) ($r['checks'] ?? 0),
|
||||
'renewals' => (int) ($r['renewals'] ?? 0),
|
||||
'failures' => (int) ($r['failures'] ?? 0),
|
||||
'strikes' => (int) ($r['strikes'] ?? 0),
|
||||
'expires' => $r['last_expiry'] ?? null,
|
||||
'last_renewal' => $r['last_renewal'] ?? null,
|
||||
'provider' => $r['provider'] ?? null,
|
||||
'state' => $removed ? 'removed' : ($retired ? 'retired' : 'active'),
|
||||
];
|
||||
$tot['tracked']++;
|
||||
$tot[$removed ? 'removed' : ($retired ? 'retired' : 'active')]++;
|
||||
$tot['renewals'] += (int) ($r['renewals'] ?? 0);
|
||||
$tot['failures'] += (int) ($r['failures'] ?? 0);
|
||||
$tot['checks'] += (int) ($r['checks'] ?? 0);
|
||||
if ($tot['oldest'] === null || $first < $tot['oldest']) $tot['oldest'] = $first;
|
||||
}
|
||||
// Longest-tracked first: the domains with the most history are the ones the card is for.
|
||||
usort($rows, fn($a, $b) => $a['first_seen'] <=> $b['first_seen']);
|
||||
$tot['oldest_span'] = $tot['oldest'] ? vv_cert_span_php($tot['oldest'], $now) : '—';
|
||||
|
||||
// DDNS is on this tab because it is the other half of the same story: a certificate is issued
|
||||
// against a name, and the name only points here while DDNS keeps it pointed. The ten dead
|
||||
// certificates removed on 2026-08-15 all failed with NXDOMAIN.
|
||||
$hostUp = strtoupper(vv_detect_host());
|
||||
$names = vv_parse_bash_array(vv_read_conf_raw(strtolower($hostUp) . '.conf'),
|
||||
$hostUp . '_DDNS_CONTAINERS');
|
||||
$running = [];
|
||||
foreach (vv_docker_containers() as $c) $running[$c['name']] = $c['status'];
|
||||
$ddns = [];
|
||||
foreach ($names as $n) {
|
||||
$n = trim((string) $n);
|
||||
if ($n === '') continue;
|
||||
$ddns[] = ['name' => $n,
|
||||
'running' => isset($running[$n]),
|
||||
'status' => $running[$n] ?? 'not running'];
|
||||
}
|
||||
|
||||
echo json_encode(['ok' => true, 'rows' => $rows, 'totals' => $tot, 'ddns' => $ddns,
|
||||
'last_pass' => $hist['last_pass'] ?? null,
|
||||
'strike_limit' => (int) (vv_conf_vars()['CERT_HISTORY_STRIKES'] ?? 5)]);
|
||||
exit;
|
||||
}
|
||||
|
||||
// Same shape as the tool's own formatter — years, months and days, because "3 years 6 months and
|
||||
// 22 days" is how the question gets asked and 1298 days is the same fact nobody thinks in.
|
||||
function vv_cert_span_php(int $from, int $to): string {
|
||||
if ($to < $from) return '0d';
|
||||
$d = (new DateTime())->setTimestamp($from)->diff((new DateTime())->setTimestamp($to));
|
||||
$out = [];
|
||||
if ($d->y) $out[] = $d->y . 'y';
|
||||
if ($d->m) $out[] = $d->m . 'mo';
|
||||
if ($d->d || !$out) $out[] = $d->d . 'd';
|
||||
return implode(' ', $out);
|
||||
}
|
||||
|
||||
// ── Read configured domains (without running checks) ─────────────────────────
|
||||
if ($action === 'domains') {
|
||||
$hostId = vv_detect_host();
|
||||
@@ -76,13 +245,27 @@ if ($action === 'domains') {
|
||||
|
||||
// ── Run cert_monitor.sh now ───────────────────────────────────────────────────
|
||||
if ($action === 'run') {
|
||||
// POST only. This executes a script, and Unraid's CSRF prepend validates POSTs while
|
||||
// ignoring GETs entirely — over GET it would run with no token check at all.
|
||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
http_response_code(405);
|
||||
echo json_encode(['ok' => false, 'error' => 'POST only']);
|
||||
exit;
|
||||
}
|
||||
$script = SCRIPTS_DIR . '/Monitors/cert_monitor.sh';
|
||||
if (!file_exists($script)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'cert_monitor.sh not found']);
|
||||
exit;
|
||||
}
|
||||
set_time_limit(180);
|
||||
exec('bash ' . escapeshellarg($script) . ' 2>&1', $out, $rc);
|
||||
// set_time_limit() does not cover exec() time on Linux, so the bound has to be external —
|
||||
// cert_monitor.sh reaches out to every configured domain and one unreachable host would
|
||||
// otherwise hold a php-fpm worker open indefinitely.
|
||||
set_time_limit(210);
|
||||
exec('timeout 180 bash ' . escapeshellarg($script) . ' 2>&1', $out, $rc);
|
||||
if ($rc === 124) {
|
||||
echo json_encode(['ok' => false, 'error' => 'cert_monitor.sh timed out after 180s']);
|
||||
exit;
|
||||
}
|
||||
// Read freshly written cache
|
||||
$data = file_exists($cacheFile)
|
||||
? (json_decode(file_get_contents($cacheFile), true) ?: null)
|
||||
@@ -125,5 +308,14 @@ if (!file_exists($cacheFile)) {
|
||||
exit;
|
||||
}
|
||||
|
||||
// ── Why renewals failed ───────────────────────────────────────────────────────
|
||||
// Reads certbot's own logs and names the categories the failures fall into. A read, so GET: it
|
||||
// opens files and nothing else. Bounded inside vv_cert_triage() by file count and bytes per file,
|
||||
// because the log directory here is 639 MB and a page request must not depend on its size.
|
||||
if ($action === 'triage') {
|
||||
echo json_encode(vv_cert_triage());
|
||||
exit;
|
||||
}
|
||||
|
||||
$data = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||
echo json_encode(array_merge(['ok' => true], $data));
|
||||
|
||||
+159
-10
@@ -1,4 +1,83 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Setup checklist. Evaluates what this host still needs before it is fully configured —
|
||||
// identity, conf, API and SSH keys, service discovery, media keys, and partnership state —
|
||||
// and names the action that fixes each gap.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// Derived, never stored. Every item is computed from the conf files and the setup state on
|
||||
// each request, so the checklist cannot go stale or disagree with reality. There is no
|
||||
// "completed" flag anyone could tick.
|
||||
//
|
||||
// The item list is not fixed. Emby and Jellyfin checks appear only once their container is
|
||||
// configured; the master.conf pull appears only on a partner host; partnership appears only
|
||||
// once a HOST2 is named. A checklist that listed everything would tell a single-host
|
||||
// install it was permanently incomplete.
|
||||
//
|
||||
// Items carry an action name, not a URL. The page maps create_key, ssh_setup, run_populate,
|
||||
// pull_master and onboard onto the right endpoint or instruction — so this file describes
|
||||
// what is wrong, and the UI owns how to fix it.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Each check answers the narrowest useful question.
|
||||
// The SSH item tests that the configured path exists on disk, not merely that a path is
|
||||
// set — a path set to a file that was never generated is the actual failure mode, and
|
||||
// the detail text distinguishes the two cases.
|
||||
//
|
||||
// Auto-populate is satisfied by any one service.
|
||||
// The check passes on the first arr key or media container found. Requiring all of them
|
||||
// would leave the item permanently red on a host that legitimately runs only some.
|
||||
//
|
||||
// Partnership reports its phases separately.
|
||||
// Phase 1 done with phase 2 outstanding is its own message, because the fix is to go
|
||||
// finish onboarding on the other host — not to re-run anything here.
|
||||
//
|
||||
// State keys are read case-insensitively.
|
||||
// Both HOST2_PHASE1_DONE and host2_phase1_done are accepted, because the state file has
|
||||
// been written by both the shell layer and the PHP layer over its life.
|
||||
//
|
||||
// complete is derived from the items, not tracked.
|
||||
// A single strict in_array(false, …) over the item results, so the summary can never
|
||||
// disagree with the list it summarises.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Read-only. This endpoint diagnoses and never fixes — every remedy is a separate,
|
||||
// explicitly invoked action. That separation is what makes it safe to poll.
|
||||
//
|
||||
// No input at all. There are no parameters, so there is nothing to validate and no way to
|
||||
// ask about a host other than this one.
|
||||
//
|
||||
// An unidentified host degrades to a report rather than an error.
|
||||
// vv_detect_host() returning 'unknown' is handled at every use — the host conf is not
|
||||
// read, and the identity and host_conf items say so explicitly. That is the exact state
|
||||
// a fresh install is in, and it is the checklist's job to describe it.
|
||||
//
|
||||
// Every conf read is defaulted.
|
||||
// vv_read_conf_raw() returns empty for a missing file and every scalar lookup is
|
||||
// trimmed with a ?? fallback, so a partial or absent conf yields items marked not-ok
|
||||
// rather than a fatal that would blank the whole panel.
|
||||
//
|
||||
// Key presence is reported, key values never are.
|
||||
// The API, SSH, Emby and Jellyfin items report only whether a value is set — and for
|
||||
// SSH, the basename of the path. No credential is returned.
|
||||
//
|
||||
// Missing is reported as missing, never as fine.
|
||||
// Every item defaults to ok:false and is only set true by a positive test. A check that
|
||||
// cannot run reports the gap it could not rule out.
|
||||
//
|
||||
// REQUEST
|
||||
// GET, no parameters
|
||||
//
|
||||
// RESPONSE
|
||||
// {"ok":true,"complete":bool,"host_id":"host<n>|unknown",
|
||||
// "items":[{"id","label","ok","detail","action"?}, …]}
|
||||
// action is present and non-null only when there is a remedy the UI can invoke.
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_parse_conf_scalar(),
|
||||
// vv_setup_state_read(), CONF_DIR
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/config.php';
|
||||
|
||||
@@ -12,11 +91,22 @@ $items = [];
|
||||
// ── Identity ──────────────────────────────────────────────────────────────────
|
||||
preg_match('/^\s*HOST1\s*=\s*"([^"]*)"/m', $master, $m1);
|
||||
$host1 = trim($m1[1] ?? '');
|
||||
// "Server identity" is this server's, not HOST1's. It was hardcoded to read master.conf's HOST1
|
||||
// and print "HOST1: <name>", so on the mirror the row that answers "which host am I" confidently
|
||||
// named the other machine — and reported ok purely because the owner's slot was filled in, which
|
||||
// says nothing about whether this host resolved to a slot at all.
|
||||
$myName = '';
|
||||
if ($hostId !== 'unknown') {
|
||||
preg_match('/^\s*' . preg_quote($hostIdUp, '/') . '\s*=\s*"([^"]*)"/m', $master, $mSelf);
|
||||
$myName = trim($mSelf[1] ?? '');
|
||||
}
|
||||
$items[] = [
|
||||
'id' => 'identity',
|
||||
'label' => 'Server identity',
|
||||
'ok' => !empty($host1),
|
||||
'detail' => $host1 ? "HOST1: $host1" : 'HOST1 blank in master.conf',
|
||||
'ok' => $hostId !== 'unknown' && !empty($myName),
|
||||
'detail' => ($hostId !== 'unknown' && $myName)
|
||||
? "$hostIdUp: $myName"
|
||||
: 'This host does not match any HOST* entry in master.conf',
|
||||
];
|
||||
|
||||
// ── Host conf ─────────────────────────────────────────────────────────────────
|
||||
@@ -97,18 +187,41 @@ if (!empty($jfContainer)) {
|
||||
];
|
||||
}
|
||||
|
||||
// ── master.conf pull (partner servers only) ───────────────────────────────────────────────────
|
||||
// ── master.conf delivery (partner servers only) ───────────────────────────────────────────────
|
||||
//
|
||||
// The question is whether this host HAS the owner's master.conf, not whether it went and
|
||||
// fetched one. There are two ways it arrives and only one of them used to count:
|
||||
//
|
||||
// pull — this host ran 'Pull from HOST1', which sets master_conf_pulled in the setup state.
|
||||
// push — HOST1 sent it during Phase 1 of onboarding, before this host even had Varaverk
|
||||
// installed. Nothing on this side runs, so no flag is written here.
|
||||
//
|
||||
// Keying solely off the pull flag meant a node whose conf had been seeded by Phase 1 —
|
||||
// populated, correct, naming both hosts — was told "Not yet pulled from HOST1" and offered a
|
||||
// button to fetch what it already had.
|
||||
//
|
||||
// The push leaves its own evidence: Phase 1 ends by writing <THIS_HOST>_PHASE1_DONE into the
|
||||
// owner's setup state and pushing that file here, so the flag can only be present on this
|
||||
// machine because the owner completed a push AT this machine. That is the discriminator.
|
||||
//
|
||||
// A populated HOST1 line is deliberately NOT the test. The wizard's manual partner path writes
|
||||
// HOST1 and HOST2 into the LOCAL template conf from what the operator typed, which would make
|
||||
// a template look delivered and hide the one button that fixes it.
|
||||
if ($hostId !== 'host1' && $hostId !== 'unknown') {
|
||||
$state = vv_setup_state_read();
|
||||
$pulled = !empty($state['master_conf_pulled']);
|
||||
$state = vv_setup_state_read();
|
||||
$pulled = !empty($state['master_conf_pulled']);
|
||||
// Both spellings, for the same reason the partnership item below accepts both.
|
||||
$pushed = !empty($state[$hostIdUp . '_PHASE1_DONE']) || !empty($state[$hostId . '_phase1_done']);
|
||||
$items[] = [
|
||||
'id' => 'master_conf',
|
||||
'label' => 'master.conf',
|
||||
'ok' => $pulled,
|
||||
'ok' => $pulled || $pushed,
|
||||
'detail' => $pulled
|
||||
? 'Synced from HOST1'
|
||||
: ($host1 ? "Not yet pulled from $host1" : 'HOST1 hostname not set in master.conf'),
|
||||
'action' => (!$pulled && $host1) ? 'pull_master' : null,
|
||||
? "Pulled from $host1"
|
||||
: ($pushed
|
||||
? "Seeded by $host1 — identity read from it"
|
||||
: ($host1 ? "Not yet pulled from $host1" : 'HOST1 hostname not set in master.conf')),
|
||||
'action' => (!$pulled && !$pushed && $host1) ? 'pull_master' : null,
|
||||
];
|
||||
}
|
||||
|
||||
@@ -130,6 +243,42 @@ if (!empty($host2)) {
|
||||
];
|
||||
}
|
||||
|
||||
$allOk = !in_array(false, array_column($items, 'ok'), true);
|
||||
// ── Blocking vs deferrable ────────────────────────────────────────────────────
|
||||
// Setup used to be complete only when every item was green, which made a media-server API key a
|
||||
// gate on finishing onboarding. A mirror that runs no Emby, or runs one the operator has not got
|
||||
// round to configuring, could never reach "complete" — and the checklist is what tells them
|
||||
// whether they are done.
|
||||
//
|
||||
// Blocking items are the ones the mesh genuinely cannot work without: who this host is, its conf,
|
||||
// its SSH key, the owner's master.conf, and the partnership itself. Everything else improves the
|
||||
// install without being load-bearing, and can be dismissed with a recorded decision.
|
||||
//
|
||||
// Deferring is per item and reversible, stored in the setup state so it survives a reload. The
|
||||
// item still shows — amber, "deferred" — rather than disappearing, because a dismissed item is a
|
||||
// decision to revisit, not a thing that stopped being true.
|
||||
$deferrable = ['api_key' => true, 'populated' => true, 'emby_key' => true, 'jellyfin_key' => true];
|
||||
$state = vv_setup_state_read();
|
||||
|
||||
foreach ($items as &$item) {
|
||||
$canDefer = !empty($deferrable[$item['id']]);
|
||||
$item['blocking'] = !$canDefer;
|
||||
$item['deferred'] = $canDefer && !$item['ok']
|
||||
&& !empty($state['DEFER_' . strtoupper($item['id'])]);
|
||||
if ($item['deferred']) {
|
||||
$item['detail'] = ($item['detail'] ?? '') . ' — deferred';
|
||||
$item['action'] = 'undefer';
|
||||
} elseif ($canDefer && !$item['ok']) {
|
||||
// Keep the item's real action as the primary; the UI offers defer alongside it.
|
||||
$item['can_defer'] = true;
|
||||
}
|
||||
}
|
||||
unset($item);
|
||||
|
||||
// Complete when every blocking item is green and every deferrable one is green or dismissed.
|
||||
$allOk = true;
|
||||
foreach ($items as $i) {
|
||||
if ($i['ok']) continue;
|
||||
if ($i['blocking'] || empty($i['deferred'])) { $allOk = false; break; }
|
||||
}
|
||||
|
||||
echo json_encode(['ok' => true, 'complete' => $allOk, 'host_id' => $hostId, 'items' => $items]);
|
||||
|
||||
@@ -1,4 +1,56 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Stale lock removal. Deletes one lock file from /tmp/unraid_locks so a script whose
|
||||
// previous run died without releasing its lock can be started again from the UI.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// The manual escape hatch for the one failure the lock discipline cannot resolve on its
|
||||
// own. Scripts take a lock to prevent concurrent runs; a run killed by OOM, a reboot mid-
|
||||
// job, or kill -9 leaves the lock behind and every subsequent run refuses to start. This
|
||||
// endpoint is how a person clears that, and it is deliberately the only way — nothing
|
||||
// automatically reaps locks, because "the lock is old" and "the job is still running" are
|
||||
// not distinguishable from the file alone.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Loads nothing.
|
||||
// This is the only endpoint in the api layer that requires no include. The operation is
|
||||
// one unlink in one fixed directory; pulling in the config layer to perform it would
|
||||
// add failure modes to something that must work when other things are broken.
|
||||
//
|
||||
// Deletes exactly one lock, never sweeps.
|
||||
// There is no clear-all. Releasing every lock at once would restart the concurrent runs
|
||||
// the locks exist to prevent.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// POST only. A GET cannot delete a lock, so a stray link, prefetch, or browser history
|
||||
// entry cannot release one.
|
||||
//
|
||||
// The filename is stripped and then pattern-matched, in that order.
|
||||
// basename() removes any directory component, and the surviving name must match
|
||||
// ^[a-zA-Z0-9_\-]+\.lock$ — no dots beyond the extension, no slashes, no traversal. The
|
||||
// two together mean the composed path cannot leave /tmp/unraid_locks, and the .lock
|
||||
// suffix means nothing but a lock file is a candidate in the first place.
|
||||
//
|
||||
// Deleting an absent lock is success, not an error.
|
||||
// file_exists() is checked and @unlink() suppressed, so two clicks, or a race with the
|
||||
// script releasing its own lock, both end with the lock gone and ok:true. The caller
|
||||
// wants the lock absent; it does not care who removed it.
|
||||
//
|
||||
// Scope is one fixed directory, hardcoded here.
|
||||
// /tmp/unraid_locks is a literal, not a config value, so no conf edit or empty variable
|
||||
// can redirect this delete somewhere else.
|
||||
//
|
||||
// REQUEST
|
||||
// POST file=<name>.lock
|
||||
//
|
||||
// RESPONSE
|
||||
// {"ok":true} lock removed, or already absent
|
||||
// {"ok":false,"error":string} wrong method or a filename that failed validation
|
||||
//
|
||||
// DEPENDS ON
|
||||
// nothing — /tmp/unraid_locks is written by the shell layer's locking helper
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
echo json_encode(['ok' => false, 'error' => 'POST only']);
|
||||
|
||||
@@ -1,4 +1,71 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Orchestrator membership toggle. Comments or uncomments one script's entry inside the
|
||||
// *_SCRIPTS arrays in master.conf, so the scheduler page can take a script out of an
|
||||
// orchestrator's run list without deleting it.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// This is not the same switch as the scheduler's enable/disable. Scheduler state controls
|
||||
// whether a *cron entry* fires; this controls whether an orchestrator *calls a child
|
||||
// script* during its own run. A script can be disabled here and still run, if another
|
||||
// orchestrator lists it — unraid_api_key_renew.sh is deliberately in two arrays.
|
||||
//
|
||||
// The edit is a comment marker, not a deletion. The line stays in master.conf with its
|
||||
// arguments and its position intact, so re-enabling restores exactly what was there before
|
||||
// and a diff shows an intent change rather than a removal.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Only the first match is toggled.
|
||||
// vv_conf_toggle_script() breaks after the first array entry it matches. A script
|
||||
// listed in two orchestrators is not silently changed in both by one click.
|
||||
//
|
||||
// No match is success, not failure.
|
||||
// A script that is not in any array has nothing to toggle and the conf is already in
|
||||
// the requested state. Returning an error there would make the UI report a problem
|
||||
// where none exists.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// POST only, checked before any parameter is read.
|
||||
//
|
||||
// The id is pattern-matched and traversal-checked separately.
|
||||
// ^[a-zA-Z0-9_./\-]+\.sh$ permits the Category/script.sh form the arrays actually use,
|
||||
// so the slash cannot simply be banned — str_contains($id, '..') is therefore a second,
|
||||
// explicit check rather than something folded into the pattern.
|
||||
//
|
||||
// The id is never used as a path.
|
||||
// It reaches vv_conf_toggle_script() only as a preg_quote()d needle matched against
|
||||
// existing lines in master.conf. Nothing is opened, executed, or created from it, so a
|
||||
// value that slipped past validation still has no file to reach.
|
||||
//
|
||||
// The scope of the edit is bounded to array bodies.
|
||||
// The library tracks whether it is inside a `*_SCRIPTS=(` block and skips every line
|
||||
// outside one, so a matching string in a comment or an unrelated variable cannot be
|
||||
// rewritten.
|
||||
//
|
||||
// The write is atomic, backed up, verified and logged.
|
||||
// vv_conf_toggle_script() goes through vv_conf_edit(), the one guarded conf write path:
|
||||
// an exclusive lock, a timestamped copy into CONF_BACKUP_DIR, bash -n on the candidate,
|
||||
// tmp + rename to install it, then the installed file is sourced to prove it still loads.
|
||||
// There is no single key to read back for a commented array member, so a clean source is
|
||||
// the whole assertion. Every script sources master.conf; a truncated or unparseable write
|
||||
// here would be a system-wide outage rather than a lost toggle.
|
||||
//
|
||||
// A script in no array writes nothing at all.
|
||||
// The rewrite returns the contents unchanged, which reports success without taking a
|
||||
// backup or touching the file. "Already in the requested state" is not a write.
|
||||
//
|
||||
// REQUEST
|
||||
// POST id=<Category/script.sh> enabled=0|1
|
||||
//
|
||||
// RESPONSE
|
||||
// {"ok":true,"error":null}
|
||||
// {"ok":false,"error":"POST only"|"Invalid id"|"Failed to write master.conf"}
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/scheduler.php vv_conf_toggle_script() → vv_conf_edit() → vv_write_conf_raw()
|
||||
// Configurations/master.conf the *_SCRIPTS arrays
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/scheduler.php';
|
||||
|
||||
@@ -9,11 +76,24 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||
|
||||
$id = trim($_POST['id'] ?? '');
|
||||
$enabled = ($_POST['enabled'] ?? '0') === '1';
|
||||
// Which orchestrator's list this click came from. Optional for callers that have only one, but
|
||||
// the scheduler always sends it: without it the toggle acts on whichever array declares the
|
||||
// script first, which for a script listed in three is right by luck at best.
|
||||
$array = trim($_POST['array'] ?? '');
|
||||
|
||||
if (!$id || !preg_match('/^[a-zA-Z0-9_.\/\-]+\.sh$/', $id) || str_contains($id, '..')) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid id']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$ok = vv_conf_toggle_script($id, $enabled);
|
||||
// Shaped like the arrays it may name and nothing else. It is compared against array names read
|
||||
// out of master.conf rather than used to build a pattern, but a value that cannot be an array
|
||||
// name has no legitimate target and is refused rather than quietly ignored — silently falling
|
||||
// back to first-match is how this went wrong in the first place.
|
||||
if ($array !== '' && !preg_match('/^[A-Z][A-Z0-9_]*_SCRIPTS$/', $array)) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid array']);
|
||||
exit;
|
||||
}
|
||||
|
||||
$ok = vv_conf_toggle_script($id, $enabled, $array !== '' ? $array : null);
|
||||
echo json_encode(['ok' => $ok, 'error' => $ok ? null : 'Failed to write master.conf']);
|
||||
|
||||
@@ -1,9 +1,135 @@
|
||||
<?php
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
// PURPOSE
|
||||
// Structured conf editing. GET returns the configuration fields relevant to one script,
|
||||
// grouped and typed for rendering as a form; POST writes a set of field changes back to
|
||||
// whichever conf files they belong to.
|
||||
//
|
||||
// OPERATIONAL MODEL
|
||||
// The counterpart to rawconf.php. That one hands over a text file; this one presents the
|
||||
// subset of keys a given script actually reads, with their types and current values, so a
|
||||
// threshold can be changed without opening master.conf and finding it.
|
||||
//
|
||||
// Changes are keyed by file, not by form. Each change carries its own target file, because
|
||||
// one script's settings routinely span master.conf and a host conf — a threshold is shared,
|
||||
// the credential it applies to is not. A single save therefore writes to several files, and
|
||||
// reports per-file results.
|
||||
//
|
||||
// A master.conf write is followed by a push to every partner, matching rawconf.php. The
|
||||
// two endpoints edit the same file and must distribute it the same way.
|
||||
//
|
||||
// DESIGN PRINCIPLES
|
||||
// Which fields belong to a script is derived, not configured.
|
||||
// vv_conf_fields_for_script() resolves them from the script itself, so a new conf
|
||||
// variable appears in the form as soon as the script reads it — there is no second list
|
||||
// to keep in step.
|
||||
//
|
||||
// Validation is total before any write begins.
|
||||
// Every change in the set is checked first, and the endpoint exits on the first bad
|
||||
// one. A partially applied save across multiple conf files is far harder to reason
|
||||
// about than a rejected one.
|
||||
//
|
||||
// ok reflects the whole set.
|
||||
// ok is false if any file failed, while files carries the per-file detail. A caller
|
||||
// that checks only ok is correct but coarse; one that wants to know which file failed
|
||||
// can see it.
|
||||
//
|
||||
// OPERATIONAL SAFEGUARDS
|
||||
// Every change names its own file, and every one is checked against the allowlist.
|
||||
// in_array(..., true) against vv_get_conf_files() per change — not once for the batch.
|
||||
// The allowlist is what enforces sparse checkout: HOST2 cannot be handed a change
|
||||
// targeting host1.conf, because host1.conf is not in its list.
|
||||
//
|
||||
// Keys must look like shell variables.
|
||||
// ^[A-Z_][A-Z0-9_]*$ — no lowercase, no punctuation, no leading digit. The key is used
|
||||
// to locate and rewrite an assignment in a bash file, so anything that could not be a
|
||||
// variable name has no legitimate target.
|
||||
//
|
||||
// Malformed payloads are refused, not coerced.
|
||||
// is_array() on the decoded changes, and an explicit missing-id check. A truncated body
|
||||
// becomes a rejection rather than an empty change set that would report success while
|
||||
// writing nothing.
|
||||
//
|
||||
// The GET path blocks traversal on the id.
|
||||
// An explicit '..' check before the id reaches the field resolver.
|
||||
//
|
||||
// Writes are atomic per file — vv_conf_write_changes() goes through the same tmp + rename
|
||||
// path as every other conf write, so a script sourcing a conf mid-save sees the old file or
|
||||
// the new one.
|
||||
//
|
||||
// The push only happens after master.conf is confirmed written.
|
||||
// Guarded on the per-file result being exactly true, so a failed edit cannot distribute
|
||||
// a stale or partly-written master.conf to partners.
|
||||
//
|
||||
// Unknown methods are refused explicitly, so a PUT or DELETE cannot fall through the two
|
||||
// handled blocks into an empty 200.
|
||||
//
|
||||
// Narrower than rawconf.php, but not narrow enough to skip the syntax gate.
|
||||
// Every write rewrites the value of an existing, named key — no key can be added,
|
||||
// deleted, or moved. But only the scalar path escapes its value: the array,
|
||||
// array_single and assoc_array paths splice the caller's text into the file verbatim,
|
||||
// and the type is chosen by the request. vv_conf_write_changes() therefore runs the
|
||||
// same `bash -n` check config.php and rawconf.php apply, and a file that does not parse
|
||||
// is reported as a failed write with the original left intact.
|
||||
//
|
||||
// REQUEST
|
||||
// GET ?id=<Category/name.sh>
|
||||
// POST id=<Category/name.sh> changes=<JSON array of {file, key, value}>
|
||||
//
|
||||
// RESPONSE
|
||||
// GET {"ok":true,"groups":[…]}
|
||||
// POST {"ok":bool,"files":{"<conf>":bool, …},"push":[{"host","ok","ready","error"}, …]}
|
||||
// {"ok":false,"error":"Invalid id"|"Missing id"|"Invalid changes"
|
||||
// |"Unauthorized file: …"|"Invalid key: …"|"Method not allowed"}
|
||||
//
|
||||
// DEPENDS ON
|
||||
// include/confform.php vv_conf_fields_for_script(), vv_conf_write_changes()
|
||||
// include/config.php vv_get_conf_files(), vv_push_master_conf(), vv_push_setup_state()
|
||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
header('Content-Type: application/json');
|
||||
require_once dirname(__DIR__) . '/include/scheduler.php';
|
||||
require_once dirname(__DIR__) . '/include/confform.php';
|
||||
|
||||
if ($_SERVER['REQUEST_METHOD'] === 'GET') {
|
||||
// Section-scoped read. A page that owns a subject rather than a script — the AI tab, and
|
||||
// partnership before it — wants the sections whose header names that subject, across every
|
||||
// conf file it is allowed to see. Same fields, same shape, same write path back; only the
|
||||
// question of "which fields" differs, so it is a mode here rather than a second endpoint
|
||||
// with its own copy of the allowlist and the master push.
|
||||
$match = trim($_GET['sections'] ?? '');
|
||||
if ($match !== '') {
|
||||
// Whole word, case-insensitive. A plain substring is far too loose on these headers —
|
||||
// "ai" alone also selects Maintenance, Containers, Failover and Arr Failed/Stalled
|
||||
// Recovery, which is nine wrong sections out of twenty-one and every one of them looks
|
||||
// deliberate once it is on the page.
|
||||
//
|
||||
// preg_quote first: the needle arrives from a query string, so it is matched as a literal
|
||||
// with boundaries around it rather than as a pattern a caller could widen to everything.
|
||||
// "*" is every section this host has. Deliberately not the same as an empty needle, which
|
||||
// would match everything by accident rather than on purpose.
|
||||
// Several words, pipe-separated, each still matched whole and still quoted as a literal.
|
||||
// One word was not enough: the Arr sections are named Arr Sync, Lidarr, Sonarr and
|
||||
// Radarr, and no single stem catches those without also catching "Array Start", which is
|
||||
// the Unraid disk array and nothing to do with them.
|
||||
$all = ($match === '*');
|
||||
$re = $all ? '' : '/\b(' . implode('|', array_map(
|
||||
fn($w) => preg_quote(trim($w), '/'),
|
||||
array_filter(explode('|', $match), fn($w) => trim($w) !== ''))) . ')\b/i';
|
||||
$out = [];
|
||||
foreach (vv_get_conf_files() as $f) {
|
||||
foreach (vv_conf_all_groups($f) as $g) {
|
||||
$name = (string) ($g['subsection'] ?? '');
|
||||
// Excluded everywhere, not only from the catch-all. A section kept out of forms
|
||||
// because a text box is the wrong control for it does not become the right
|
||||
// control because a different page asked.
|
||||
if (isset(VV_UI_SECTION_EXCLUDE[$name])) continue;
|
||||
if ($all || preg_match($re, $name)) $out[] = $g;
|
||||
}
|
||||
}
|
||||
echo json_encode(['ok' => true, 'groups' => $out]);
|
||||
exit;
|
||||
}
|
||||
|
||||
$id = trim($_GET['id'] ?? '');
|
||||
if (!$id || str_contains($id, '..')) {
|
||||
echo json_encode(['ok' => false, 'error' => 'Invalid id']);
|
||||
@@ -18,7 +144,10 @@ if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
$id = trim($_POST['id'] ?? '');
|
||||
$rawJson = $_POST['changes'] ?? '[]';
|
||||
|
||||
if (!$id) { echo json_encode(['ok' => false, 'error' => 'Missing id']); exit; }
|
||||
// Optional. It labels which script's form was open and is used nowhere in the write — every
|
||||
// change already names its own file and key, and those are what is validated below. A
|
||||
// section-scoped save has no script to name, and inventing one so this check would pass
|
||||
// would be a guard that only ever guarded against itself.
|
||||
|
||||
$changes = json_decode($rawJson, true);
|
||||
if (!is_array($changes)) { echo json_encode(['ok' => false, 'error' => 'Invalid changes']); exit; }
|
||||
@@ -35,7 +164,13 @@ if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
}
|
||||
}
|
||||
|
||||
$results = vv_conf_write_changes($changes);
|
||||
// Declared before it is passed. It is a by-reference array parameter, and an undefined
|
||||
// variable arrives there as null — which under PHP 8 is a TypeError thrown before a single
|
||||
// byte is written, so every save through this endpoint died with a 500 and the page saw an
|
||||
// unparseable response rather than a refusal it could report. The callers that pass no
|
||||
// second argument were never affected, which is why it survived: this is the only one.
|
||||
$rejected = [];
|
||||
$results = vv_conf_write_changes($changes, $rejected);
|
||||
|
||||
// Propagate master.conf to partner hosts when the owner edits it (mirrors rawconf.php).
|
||||
$push = [];
|
||||
@@ -44,7 +179,18 @@ if ($_SERVER['REQUEST_METHOD'] === 'POST') {
|
||||
vv_push_setup_state();
|
||||
}
|
||||
|
||||
echo json_encode(['ok' => !in_array(false, $results, true), 'files' => $results, 'push' => $push]);
|
||||
// A refused change never reaches a file, so it leaves no false in $results — a save whose
|
||||
// only change was refused used to answer ok:true and show the operator their old value back
|
||||
// with no explanation. Refusals are failures here and they are named: the whole point of the
|
||||
// path guard is that someone learns their edit would have widened a delete target.
|
||||
echo json_encode(['ok' => !in_array(false, $results, true) && !$rejected,
|
||||
'files' => $results,
|
||||
'rejected' => $rejected,
|
||||
'error' => $rejected
|
||||
? 'Refused: ' . implode(', ',
|
||||
array_map(fn($r) => $r['key'] . ' (' . $r['reason'] . ')', $rejected))
|
||||
: null,
|
||||
'push' => $push]);
|
||||
exit;
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user