Compare commits
211
Commits
187dc188c2
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5fa7750046 | ||
|
|
1838eed855 | ||
|
|
96d8a5e3f0 | ||
|
|
0431e720de | ||
|
|
7f4921de49 | ||
|
|
d4c19baa32 | ||
|
|
d444fd8036 | ||
|
|
d42b1e2dda | ||
|
|
9492dc4c39 | ||
|
|
172beca3c5 | ||
|
|
f0c1289519 | ||
|
|
128172d3a8 | ||
|
|
820e8325a6 | ||
|
|
74c6a0f5eb | ||
|
|
d289a9101c | ||
|
|
9553ecb16a | ||
|
|
fb104bf05a | ||
|
|
f4adc31215 | ||
|
|
c6254f2342 | ||
|
|
d25b147a56 | ||
|
|
1f9915f074 | ||
|
|
c0e7a97a56 | ||
|
|
68c835f33b | ||
|
|
6b9fda76db | ||
|
|
1be259b66d | ||
|
|
2c3c88d067 | ||
|
|
3fc14d16bf | ||
|
|
4cc50889d6 | ||
|
|
63e68740b6 | ||
|
|
f2fddeb87d | ||
|
|
eda411c0be | ||
|
|
492757ce70 | ||
|
|
b58282b3d6 | ||
|
|
9da2760a42 | ||
|
|
049f633667 | ||
|
|
84946ed0c6 | ||
|
|
514e13660c | ||
|
|
9333334b7b | ||
|
|
367d0203b8 | ||
|
|
37017882c7 | ||
|
|
f116a1584e | ||
|
|
1a5bacce87 | ||
|
|
f6e5f7dcd6 | ||
|
|
0f0c407458 | ||
|
|
354992c9e7 | ||
|
|
28987240aa | ||
|
|
fbf2b92167 | ||
|
|
206a119a4b | ||
|
|
671e7ea5a4 | ||
|
|
d5cf3db2ec | ||
|
|
dc8823724d | ||
|
|
a392108562 | ||
|
|
fd1ab58598 | ||
|
|
bd13bdc6e1 | ||
|
|
bdf1bce4be | ||
|
|
c5c0477653 | ||
|
|
76faf0f9f8 | ||
|
|
879109e55d | ||
|
|
61b7d2a96a | ||
|
|
15d802ae3d | ||
|
|
ac17be2cd9 | ||
|
|
f1ba418aa8 | ||
|
|
b17cc3a09d | ||
|
|
1c38abe1d0 | ||
|
|
5d42a28d34 | ||
|
|
e5f4ede3f2 | ||
|
|
ce613bcf9d | ||
|
|
8ce8f0dfbe | ||
|
|
13c221244e | ||
|
|
a89b704fa1 | ||
|
|
0f228020ef | ||
|
|
8454d3d0a2 | ||
|
|
da15cc7531 | ||
|
|
5924aae1c0 | ||
|
|
46239012b4 | ||
|
|
f8119f3786 | ||
|
|
41c0598220 | ||
|
|
387382ec9f | ||
|
|
20998c8c31 | ||
|
|
f5d6edd9d0 | ||
|
|
4808a57289 | ||
|
|
96b907acbc | ||
|
|
d9c37ac763 | ||
|
|
146be2d0ad | ||
|
|
906375dc23 | ||
|
|
e1cdf03d40 | ||
|
|
8041da5424 | ||
|
|
7c4f881b57 | ||
|
|
81abc5adee | ||
|
|
331b7b6b13 | ||
|
|
fe23c59e0c | ||
|
|
666e1ebab2 | ||
|
|
3d05276312 | ||
|
|
00fdee5649 | ||
|
|
4a29e7bc99 | ||
|
|
a07019e3aa | ||
|
|
aa16fbaa8b | ||
|
|
9c15df8e22 | ||
|
|
111c70d6b4 | ||
|
|
0621f66889 | ||
|
|
4accd6b67e | ||
|
|
63ec110177 | ||
|
|
a27bb41b42 | ||
|
|
eaa78452bc | ||
|
|
4eb0a9c608 | ||
|
|
2e646f82b4 | ||
|
|
74bd40ba78 | ||
|
|
4f99efcf2c | ||
|
|
be10e78c7a | ||
|
|
b6a677a710 | ||
|
|
53d7784d5d | ||
|
|
10f2a0a8a4 | ||
|
|
eab5cc911c | ||
|
|
d88fe381db | ||
|
|
ecded4a8a7 | ||
|
|
772a3bdce2 | ||
|
|
fbf6a07da9 | ||
|
|
25e055a7ac | ||
|
|
e024a29157 | ||
|
|
83f3b7e226 | ||
|
|
d3d6f30f72 | ||
|
|
99c94ffaec | ||
|
|
1a836dac75 | ||
|
|
0447fa6f86 | ||
|
|
e8ee5b052b | ||
|
|
d7980da35e | ||
|
|
239e100b44 | ||
|
|
2216072669 | ||
|
|
8cf3ff31db | ||
|
|
b108a6fe62 | ||
|
|
f58ff68ab3 | ||
|
|
e7e9d9fecf | ||
|
|
d417bd5360 | ||
|
|
90f3b0b7da | ||
|
|
bce3bf61e9 | ||
|
|
49372996ac | ||
|
|
e3ed5a53f2 | ||
|
|
1231cd69a8 | ||
|
|
8e85a67e7b | ||
|
|
1badb93331 | ||
|
|
e23358ea0c | ||
|
|
73bbae14c1 | ||
|
|
74e3b8b067 | ||
|
|
9a1254048e | ||
|
|
7ae510ed28 | ||
|
|
620d7eb6f1 | ||
|
|
cd21fca749 | ||
|
|
ead5d38d7c | ||
|
|
c4bfa6bdfe | ||
|
|
d8a77c700b | ||
|
|
5fc53948c6 | ||
|
|
587440b6bc | ||
|
|
fb49eb20e8 | ||
|
|
60e87ba518 | ||
|
|
ff94ccb343 | ||
|
|
42711ff9ac | ||
|
|
7c957d96be | ||
|
|
427a31cdff | ||
|
|
51125fa05a | ||
|
|
24e347bb2d | ||
|
|
1a47f864f9 | ||
|
|
ad623353bc | ||
|
|
9d2610a911 | ||
|
|
bc9692f521 | ||
|
|
d8e3c09b57 | ||
|
|
7e7d845f61 | ||
|
|
b83c118c7f | ||
|
|
069815790b | ||
|
|
366e2a269f | ||
|
|
bdc2cdbadb | ||
|
|
fe58fb7826 | ||
|
|
6dbb076a1e | ||
|
|
3676526daf | ||
|
|
6a15a9d99a | ||
|
|
a46317de18 | ||
|
|
32355e0219 | ||
|
|
2cbc06a683 | ||
|
|
cce1e25c2b | ||
|
|
39dba3f8ab | ||
|
|
5bdf3bff60 | ||
|
|
65516ea3ac | ||
|
|
27989d066e | ||
|
|
3ae6298656 | ||
|
|
662f4f0d53 | ||
|
|
a02b918ff9 | ||
|
|
23825a824d | ||
|
|
cf67393db6 | ||
|
|
7ed0f5c23b | ||
|
|
f363c6cd55 | ||
|
|
c8a6f58007 | ||
|
|
93cdbaac57 | ||
|
|
47ad3b3075 | ||
|
|
1493cf2dcf | ||
|
|
35e59d2510 | ||
|
|
4f95cc6d13 | ||
|
|
f627658567 | ||
|
|
3c24550b19 | ||
|
|
98fab394a1 | ||
|
|
6fbc2b4125 | ||
|
|
64de77c065 | ||
|
|
19ba1e723c | ||
|
|
1d4dc44668 | ||
|
|
0fe3435033 | ||
|
|
6d9c856514 | ||
|
|
9634838f64 | ||
|
|
28273ffc2a | ||
|
|
7b65b64864 | ||
|
|
b90fd59544 | ||
|
|
d5ee2d7ef1 | ||
|
|
73358e6bae | ||
|
|
f2ad41e8e2 |
+12
@@ -43,3 +43,15 @@ Plugin/dist/
|
|||||||
*.swp
|
*.swp
|
||||||
*~
|
*~
|
||||||
.vscode/
|
.vscode/
|
||||||
|
|
||||||
|
# ── Local-only plugin surfaces (per-installation, never pushed) ───────────────
|
||||||
|
# Varaverk.page discovers pages/local/*.php and registers each as a tab; api/local/ holds their
|
||||||
|
# endpoints. Both are symlinks into a store outside this repo, so what they contain belongs to
|
||||||
|
# one installation and is not part of the project. The tracked loader is deliberately generic —
|
||||||
|
# it names no page — so the public mirror never learns what a given server runs here.
|
||||||
|
#
|
||||||
|
# No trailing slash on either pattern. These paths are symlinks, not directories, and git treats
|
||||||
|
# a symlink as a blob — a "dir/" pattern does not match one, so the entries sat untracked rather
|
||||||
|
# than ignored, which is the same near-miss the *.bak rule above documents.
|
||||||
|
Plugin/unraid/pages/local
|
||||||
|
Plugin/unraid/api/local
|
||||||
|
|||||||
@@ -194,6 +194,19 @@ if [[ "${AI_ENABLED:-false}" != "true" ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# The mesh shares one AI, and the index belongs to the node that holds the model. A mirror has the
|
||||||
|
# same checkout and could build one, but nothing there would read it: retrieval happens wherever
|
||||||
|
# generation happens, which is the owner.
|
||||||
|
#
|
||||||
|
# A skip, not an error. This is reached from git_pull_execute.sh on every node after every pull;
|
||||||
|
# before the AI became mesh-wide it ran here too and failed on the empty OLLAMA_URL, nightly and
|
||||||
|
# silently, because the caller discards its output.
|
||||||
|
_ai_owner="${AI_OWNER_HOST:-host1}"
|
||||||
|
if [[ "${MY_ID,,}" != "${_ai_owner,,}" ]]; then
|
||||||
|
log "This node is not the AI owner ($_ai_owner) — the index lives there; skipping"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false && "$EUID" -ne 0 ]]; then
|
if [[ "$DRY_RUN" == false && "$EUID" -ne 0 ]]; then
|
||||||
error "Must be run as root"
|
error "Must be run as root"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -204,6 +217,14 @@ acquire_lock
|
|||||||
command -v node >/dev/null 2>&1 || { error "node not found — required to build the index"; exit 1; }
|
command -v node >/dev/null 2>&1 || { error "node not found — required to build the index"; exit 1; }
|
||||||
[[ -f "$CLI" ]] || { error "missing $CLI"; exit 1; }
|
[[ -f "$CLI" ]] || { error "missing $CLI"; exit 1; }
|
||||||
|
|
||||||
|
# The AI owner has had data/ai since the subsystem was built, so nothing ever created it — cli.js
|
||||||
|
# opens the DB by path and does not make the directory. On a first build the failure surfaces as a
|
||||||
|
# sqlite open error rather than as the missing directory it is.
|
||||||
|
if [[ "$DRY_RUN" == false ]] && ! mkdir -p "$(dirname "$DB")"; then
|
||||||
|
error "Cannot create $(dirname "$DB")"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -z "$OLLAMA_URL" ]]; then
|
if [[ -z "$OLLAMA_URL" ]]; then
|
||||||
error "${MY_ID}_OLLAMA_URL is empty — no local Ollama to index against"
|
error "${MY_ID}_OLLAMA_URL is empty — no local Ollama to index against"
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
+26
-1
@@ -30,6 +30,31 @@
|
|||||||
# fetches its own data controls its own freshness rather than depending on the partner's cron.
|
# fetches its own data controls its own freshness rather than depending on the partner's cron.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# A missing file means unknown, never zero.
|
||||||
|
# The whole point of the tab reporting "not collected here" is that it is a different claim
|
||||||
|
# from "this partner spent nothing". If a partner is dark, unreachable or has never run a turn,
|
||||||
|
# there is simply no cache file, and the reader is expected to say so rather than render a 0
|
||||||
|
# that looks like a measurement.
|
||||||
|
#
|
||||||
|
# The reader pulls; nobody pushes.
|
||||||
|
# conf_sync.sh pushes as well, because a partner that cannot reach us still needs our
|
||||||
|
# credentials. Nothing here is needed by anyone else, so a host that wants fleet totals fetches
|
||||||
|
# them and owns its own freshness instead of depending on someone else's cron having run.
|
||||||
|
#
|
||||||
|
# RAM, not flash.
|
||||||
|
# The cache lands in tmpfs. It is a copy of a file that already exists on the partner and is
|
||||||
|
# rebuilt on the next pass, so writing it to flash would cost wear for something that is never
|
||||||
|
# worth surviving a reboot.
|
||||||
|
#
|
||||||
|
# Same shape as conf_sync.sh, deliberately.
|
||||||
|
# Resolve over Tailscale, scp one small file into a tmpfs cache, let a missing file mean
|
||||||
|
# unknown. A second transport pattern for a second small file would be a second set of
|
||||||
|
# failure modes to learn.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -206,6 +231,6 @@ _summary="AI token sync complete — pulled $PULLED"
|
|||||||
info "$_summary"
|
info "$_summary"
|
||||||
|
|
||||||
# Only a partner that answered and then failed the transfer is worth an exit code. An absent
|
# Only a partner that answered and then failed the transfer is worth an exit code. An absent
|
||||||
# partner is the normal state while HOST2 is being rebuilt.
|
# partner is the normal state whenever a partner is not yet onboarded.
|
||||||
[[ "$FAILED" -gt 0 ]] && exit 1
|
[[ "$FAILED" -gt 0 ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
Executable → Regular
+120
-8
@@ -273,6 +273,9 @@ touch "$CORRUPTION_SCAN_STATE_FILE"
|
|||||||
|
|
||||||
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
|
CORRUPTION_SCAN_STRIKES_FILE="${CORRUPTION_SCAN_STRIKES_FILE:-$DATA_DIR/corruption_scan_strikes.tsv}"
|
||||||
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
|
CORRUPTION_SCAN_STRIKE_LIMIT="${CORRUPTION_SCAN_STRIKE_LIMIT:-2}"
|
||||||
|
CORRUPTION_SCAN_MAX_CORRUPT_PCT="${CORRUPTION_SCAN_MAX_CORRUPT_PCT:-10}"
|
||||||
|
CORRUPTION_SCAN_MAX_CONSECUTIVE="${CORRUPTION_SCAN_MAX_CONSECUTIVE:-15}"
|
||||||
|
CORRUPTION_SCAN_GUARD_MIN_SCANNED="${CORRUPTION_SCAN_GUARD_MIN_SCANNED:-20}"
|
||||||
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
|
mkdir -p "$(dirname "$CORRUPTION_SCAN_STRIKES_FILE")"
|
||||||
touch "$CORRUPTION_SCAN_STRIKES_FILE"
|
touch "$CORRUPTION_SCAN_STRIKES_FILE"
|
||||||
|
|
||||||
@@ -301,6 +304,19 @@ reset_scan_strikes() {
|
|||||||
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
|
[[ -n "$current" && "$current" != "0" ]] && set_scan_strikes "$1" 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Bails out of the whole run without committing anything. Safe to call at any point before
|
||||||
|
# the commit phase: strikes are queued in memory until then, so an abort leaves the strike
|
||||||
|
# file exactly as the previous run left it and deletes nothing.
|
||||||
|
abort_scan() {
|
||||||
|
local why="$1"
|
||||||
|
error "Corruption scan ABORTED — $why"
|
||||||
|
error "No strikes recorded and nothing remediated this run — the library was not trusted."
|
||||||
|
[[ -n "${FRESH_CLEAN_TMP:-}" ]] && rm -f "$FRESH_CLEAN_TMP"
|
||||||
|
notify "Corruption scan aborted on $(hostname) ($MY_ID) — $why. Nothing deleted." \
|
||||||
|
"Arr Corruption Scan" "warning"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
|
# Per-arr API shape differences — everything else in the scan/strike/remediate loop below is
|
||||||
# identical between Sonarr and Radarr.
|
# identical between Sonarr and Radarr.
|
||||||
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
|
declare -A ARR_FILE_ENDPOINT=( [sonarr]="episodefile" [radarr]="moviefile" )
|
||||||
@@ -322,6 +338,8 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
echo "$ICON_GEAR State file: $CORRUPTION_SCAN_STATE_FILE"
|
||||||
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
|
echo "$ICON_GEAR Strike limit: $CORRUPTION_SCAN_STRIKE_LIMIT"
|
||||||
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
echo "$ICON_GEAR Remediate: $REMEDIATE"
|
||||||
|
echo "$ICON_GEAR Corrupt ceiling: ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% of scanned (min ${CORRUPTION_SCAN_GUARD_MIN_SCANNED} scanned)"
|
||||||
|
echo "$ICON_GEAR Consecutive trip: $CORRUPTION_SCAN_MAX_CONSECUTIVE"
|
||||||
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
|
echo "$ICON_GEAR Scan limit: ${SCAN_LIMIT:-unlimited} (per arr)"
|
||||||
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
echo "$ICON_GEAR Path filter: ${PATH_FILTER:-none}"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
@@ -357,13 +375,36 @@ ffprobe_translate_path() {
|
|||||||
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
# Probes one file. Echoes "clean" or "corrupt:<reason>". Never trusts a truncated/garbled
|
||||||
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
# stderr as automatically corrupt — only a real non-empty ffprobe stderr counts.
|
||||||
probe_file() {
|
probe_file() {
|
||||||
local host_path="$1" container_path output
|
local host_path="$1" container_path output rc
|
||||||
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
container_path=$(ffprobe_translate_path "$host_path") || { echo "unmapped"; return; }
|
||||||
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
output=$(docker exec "$FFPROBE_CONTAINER" "$FFPROBE_BIN" -v error "$container_path" 2>&1)
|
||||||
|
rc=$?
|
||||||
|
|
||||||
|
# docker exec writes its own failures to the same stream ffprobe uses, so a stopped
|
||||||
|
# container or an unreachable daemon is otherwise indistinguishable from a corrupt
|
||||||
|
# header. A stopped container exits 1 with a daemon message; a missing binary exits
|
||||||
|
# 127 — neither is evidence about the file, so both must be caught.
|
||||||
|
if (( rc >= 125 )) \
|
||||||
|
|| [[ "$output" == "Error response from daemon:"* \
|
||||||
|
|| "$output" == "Cannot connect to the Docker daemon"* \
|
||||||
|
|| "$output" == "error during connect:"* ]]; then
|
||||||
|
echo "probe_error:${output//$'\n'/ }"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -z "$output" ]]; then
|
if [[ -z "$output" ]]; then
|
||||||
echo "clean"
|
echo "clean"
|
||||||
else
|
elif (( rc != 0 )); then
|
||||||
|
# ffprobe could not parse the file — EBML header parsing failed, moov atom not found,
|
||||||
|
# contradictionary STSC and STCO. This is the only class that may be remediated.
|
||||||
echo "corrupt:${output//$'\n'/ }"
|
echo "corrupt:${output//$'\n'/ }"
|
||||||
|
else
|
||||||
|
# Exit 0 with stderr output: a recoverable muxing complaint, most commonly
|
||||||
|
# "Referenced QT chapter track not found", which many recent .mp4 releases emit and
|
||||||
|
# which says nothing about playability. Equating any stderr with corruption is what
|
||||||
|
# produced 103 "corrupt" files on 2026-08-23 — 28 of 43 newly scanned Radarr items.
|
||||||
|
# Reported for visibility, never strike-tracked, never remediated.
|
||||||
|
echo "suspect:${output//$'\n'/ }"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -442,7 +483,7 @@ TOTAL_SCANNED=0
|
|||||||
TOTAL_CORRUPT=0
|
TOTAL_CORRUPT=0
|
||||||
TOTAL_REMEDIATED=0
|
TOTAL_REMEDIATED=0
|
||||||
TOTAL_REMEDIATE_FAILED=0
|
TOTAL_REMEDIATE_FAILED=0
|
||||||
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
declare -A ARR_SCANNED ARR_SKIPPED_CACHED ARR_SKIPPED_UNMAPPED ARR_CORRUPT ARR_SUSPECT ARR_PROBE_ERRORS ARR_STRIKE_HELD ARR_REMEDIATED ARR_REMEDIATE_FAILED
|
||||||
|
|
||||||
for arr in sonarr radarr; do
|
for arr in sonarr radarr; do
|
||||||
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
||||||
@@ -532,6 +573,12 @@ for arr in sonarr radarr; do
|
|||||||
STRIKE_HELD=0
|
STRIKE_HELD=0
|
||||||
REMEDIATED=0
|
REMEDIATED=0
|
||||||
REMEDIATE_FAILED=0
|
REMEDIATE_FAILED=0
|
||||||
|
PROBE_ERRORS=0
|
||||||
|
SUSPECT_COUNT=0
|
||||||
|
CONSECUTIVE_BAD=0
|
||||||
|
QUEUE_PATH=()
|
||||||
|
QUEUE_STRIKES=()
|
||||||
|
QUEUE_ITEM=()
|
||||||
|
|
||||||
FRESH_CLEAN_TMP=$(mktemp)
|
FRESH_CLEAN_TMP=$(mktemp)
|
||||||
|
|
||||||
@@ -566,23 +613,82 @@ for arr in sonarr radarr; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# A docker-level failure is not evidence about the file. Count it, never queue it.
|
||||||
|
if [[ "$result" == probe_error:* ]]; then
|
||||||
|
(( PROBE_ERRORS++ ))
|
||||||
|
(( CONSECUTIVE_BAD++ ))
|
||||||
|
warn " ? $host_path — probe failed, NOT counted as corrupt: ${result#probe_error:}"
|
||||||
|
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||||
|
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||||
|
fi
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$result" == "clean" ]]; then
|
if [[ "$result" == "clean" ]]; then
|
||||||
|
CONSECUTIVE_BAD=0
|
||||||
reset_scan_strikes "$host_path"
|
reset_scan_strikes "$host_path"
|
||||||
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
echo -e "${host_path}\t${stamp}" >> "$FRESH_CLEAN_TMP"
|
||||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
[[ "$ENABLE_LOGGING" == true ]] && echo " $ICON_SUCCESS $host_path"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# corrupt:<reason>
|
# A successful probe that merely warned. Proves the container is alive, so it clears
|
||||||
|
# the consecutive-failure tripwire, but it never becomes a strike.
|
||||||
|
if [[ "$result" == suspect:* ]]; then
|
||||||
|
CONSECUTIVE_BAD=0
|
||||||
|
(( SUSPECT_COUNT++ ))
|
||||||
|
[[ "$ENABLE_LOGGING" == true ]] && warn " ~ $host_path — ffprobe warning (exit 0), NOT corrupt: ${result#suspect:}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# corrupt:<reason> — queued, NOT committed. Nothing reaches the strike file and nothing
|
||||||
|
# is deleted until this arr has been fully probed and the guards below have passed. A
|
||||||
|
# container that dies mid-scan makes every remaining file read as corrupt, and a delete
|
||||||
|
# cannot be undone — so the destructive half has to wait until the corrupt rate for the
|
||||||
|
# whole run is known. 2026-08-23: one Jellyfin restart produced 103 false positives.
|
||||||
reason="${result#corrupt:}"
|
reason="${result#corrupt:}"
|
||||||
(( CORRUPT_COUNT++ ))
|
(( CORRUPT_COUNT++ ))
|
||||||
strikes=$(increment_scan_strikes "$host_path")
|
(( CONSECUTIVE_BAD++ ))
|
||||||
|
|
||||||
|
prev_strikes=$(get_scan_strikes "$host_path")
|
||||||
|
prev_strikes="${prev_strikes//[^0-9]/}"
|
||||||
|
strikes=$(( ${prev_strikes:-0} + 1 ))
|
||||||
|
|
||||||
|
QUEUE_PATH+=("$host_path")
|
||||||
|
QUEUE_STRIKES+=("$strikes")
|
||||||
|
QUEUE_ITEM+=("$item")
|
||||||
|
|
||||||
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
|
echo " $ICON_ERROR CORRUPT: $host_path (strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT)"
|
||||||
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
[[ "$ENABLE_LOGGING" == true ]] && echo " $reason"
|
||||||
|
|
||||||
if [[ "$REMEDIATE" != true ]]; then
|
if (( CONSECUTIVE_BAD >= CORRUPTION_SCAN_MAX_CONSECUTIVE )); then
|
||||||
continue
|
abort_scan "$CONSECUTIVE_BAD files in a row failed to probe cleanly"
|
||||||
fi
|
fi
|
||||||
|
done < <(echo "$ITEMS" | jq -c '.[]')
|
||||||
|
|
||||||
|
# ━━━ False-positive guards — run before anything is committed ━━━
|
||||||
|
if (( CORRUPT_COUNT > 0 )); then
|
||||||
|
# The pre-flight check only proves the container was up when the scan started.
|
||||||
|
# Re-check now: a mid-scan death is exactly what this guard exists to catch.
|
||||||
|
check_container_health "$FFPROBE_CONTAINER" "${DOCKER_TIMEOUT:-30}" "Arr Corruption Scan"
|
||||||
|
|
||||||
|
if (( SCANNED >= CORRUPTION_SCAN_GUARD_MIN_SCANNED )); then
|
||||||
|
corrupt_pct=$(( CORRUPT_COUNT * 100 / SCANNED ))
|
||||||
|
if (( corrupt_pct >= CORRUPTION_SCAN_MAX_CORRUPT_PCT )); then
|
||||||
|
abort_scan "$CORRUPT_COUNT of $SCANNED probed files (${corrupt_pct}%) read as corrupt — at or above the ${CORRUPTION_SCAN_MAX_CORRUPT_PCT}% ceiling"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ━━━ Guards passed — commit strikes, then remediate whatever reached the limit ━━━
|
||||||
|
for _q in "${!QUEUE_PATH[@]}"; do
|
||||||
|
host_path="${QUEUE_PATH[$_q]}"
|
||||||
|
strikes="${QUEUE_STRIKES[$_q]}"
|
||||||
|
item="${QUEUE_ITEM[$_q]}"
|
||||||
|
|
||||||
|
set_scan_strikes "$host_path" "$strikes"
|
||||||
|
|
||||||
|
[[ "$REMEDIATE" != true ]] && continue
|
||||||
|
|
||||||
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
|
if (( strikes < CORRUPTION_SCAN_STRIKE_LIMIT )); then
|
||||||
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
|
warn " $host_path — strike $strikes/$CORRUPTION_SCAN_STRIKE_LIMIT, not yet remediating (needs repeat confirmation)"
|
||||||
@@ -591,6 +697,8 @@ for arr in sonarr radarr; do
|
|||||||
fi
|
fi
|
||||||
reset_scan_strikes "$host_path"
|
reset_scan_strikes "$host_path"
|
||||||
|
|
||||||
|
file_id=$(echo "$item" | jq -r '.file_id')
|
||||||
|
parent_id=$(echo "$item" | jq -r '.parent_id')
|
||||||
title=$(echo "$item" | jq -r '.title')
|
title=$(echo "$item" | jq -r '.title')
|
||||||
|
|
||||||
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
http_code=$(curl -sf -o /dev/null -w "%{http_code}" -X DELETE \
|
||||||
@@ -625,7 +733,7 @@ for arr in sonarr radarr; do
|
|||||||
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
warn " $title — deleted and verified, but search trigger returned HTTP $search_code"
|
||||||
(( REMEDIATE_FAILED++ ))
|
(( REMEDIATE_FAILED++ ))
|
||||||
fi
|
fi
|
||||||
done < <(echo "$ITEMS" | jq -c '.[]')
|
done
|
||||||
|
|
||||||
merge_clean_state "$FRESH_CLEAN_TMP"
|
merge_clean_state "$FRESH_CLEAN_TMP"
|
||||||
rm -f "$FRESH_CLEAN_TMP"
|
rm -f "$FRESH_CLEAN_TMP"
|
||||||
@@ -634,6 +742,8 @@ for arr in sonarr radarr; do
|
|||||||
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
|
ARR_SKIPPED_CACHED[$arr]=$SKIPPED_CACHED
|
||||||
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
|
ARR_SKIPPED_UNMAPPED[$arr]=$SKIPPED_UNMAPPED
|
||||||
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
|
ARR_CORRUPT[$arr]=$CORRUPT_COUNT
|
||||||
|
ARR_SUSPECT[$arr]=$SUSPECT_COUNT
|
||||||
|
ARR_PROBE_ERRORS[$arr]=$PROBE_ERRORS
|
||||||
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
|
ARR_STRIKE_HELD[$arr]=$STRIKE_HELD
|
||||||
ARR_REMEDIATED[$arr]=$REMEDIATED
|
ARR_REMEDIATED[$arr]=$REMEDIATED
|
||||||
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
|
ARR_REMEDIATE_FAILED[$arr]=$REMEDIATE_FAILED
|
||||||
@@ -658,6 +768,8 @@ for arr in sonarr radarr; do
|
|||||||
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
|
echo " $ICON_SUCCESS Skipped (cached): ${ARR_SKIPPED_CACHED[$arr]}"
|
||||||
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
|
echo " $ICON_WARN Skipped (unmapped): ${ARR_SKIPPED_UNMAPPED[$arr]}"
|
||||||
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
|
echo " $ICON_ERROR Corrupt found: ${ARR_CORRUPT[$arr]}"
|
||||||
|
echo " $ICON_WARN Warnings (exit 0): ${ARR_SUSPECT[$arr]} (reported, never remediated)"
|
||||||
|
echo " $ICON_WARN Probe errors: ${ARR_PROBE_ERRORS[$arr]} (not counted as corrupt)"
|
||||||
if [[ "$REMEDIATE" == true ]]; then
|
if [[ "$REMEDIATE" == true ]]; then
|
||||||
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
|
echo " $ICON_WARN Held (strikes): ${ARR_STRIKE_HELD[$arr]}"
|
||||||
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
|
echo " $ICON_SUCCESS Remediated: ${ARR_REMEDIATED[$arr]}"
|
||||||
|
|||||||
@@ -146,10 +146,21 @@
|
|||||||
# (default: 7)
|
# (default: 7)
|
||||||
#
|
#
|
||||||
# DOWNLOAD_ORPHAN_MIN_VIDEO_MB
|
# DOWNLOAD_ORPHAN_MIN_VIDEO_MB
|
||||||
# An entry with no video file above this size is JUNK (default: 50)
|
# An entry with no video file above this size is JUNK (default: 50). Sonarr/Radarr only.
|
||||||
|
#
|
||||||
|
# DOWNLOAD_ORPHAN_MIN_AUDIO_MB
|
||||||
|
# The same test for Lidarr (default: 2). Separate because a 50M floor would mark
|
||||||
|
# every album folder as JUNK — single tracks rarely reach it.
|
||||||
|
#
|
||||||
|
# DOWNLOAD_ORPHAN_KEEP_MARKER
|
||||||
|
# A file with this name inside a download folder pins it — the folder is never
|
||||||
|
# classified or deleted (default: .vv-keep). For lossless rips the library holds
|
||||||
|
# only at lower quality, which REDUNDANT would otherwise sweep.
|
||||||
#
|
#
|
||||||
# DOWNLOAD_ORPHAN_MAX_DELETE_GB
|
# DOWNLOAD_ORPHAN_MAX_DELETE_GB
|
||||||
# Abort the delete pass if the run total exceeds this (default: 100)
|
# Per-run delete budget in GB (default: 100). A backlog above this is drained
|
||||||
|
# safest-first (JUNK, then REDUNDANT, then UNMATCHED) up to the budget, and the
|
||||||
|
# remainder is deferred to the next run rather than aborting the pass.
|
||||||
#
|
#
|
||||||
# SONARR_EXTENSIONS / RADARR_EXTENSIONS
|
# SONARR_EXTENSIONS / RADARR_EXTENSIONS
|
||||||
# Video extensions used to decide whether an entry contains real media
|
# Video extensions used to decide whether an entry contains real media
|
||||||
@@ -161,7 +172,7 @@
|
|||||||
# arr_download_orphan_cleaner.sh — daily orchestrator entry
|
# arr_download_orphan_cleaner.sh — daily orchestrator entry
|
||||||
# arr_download_orphan_cleaner.sh --dry-run — classify and report only
|
# arr_download_orphan_cleaner.sh --dry-run — classify and report only
|
||||||
# arr_download_orphan_cleaner.sh --status — show config and exit
|
# arr_download_orphan_cleaner.sh --status — show config and exit
|
||||||
# arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB cap
|
# arr_download_orphan_cleaner.sh --i-know-what-im-doing — bypass MAX_DELETE_GB budget
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
@@ -197,6 +208,8 @@ fi
|
|||||||
|
|
||||||
DOWNLOAD_ORPHAN_AGE="${DOWNLOAD_ORPHAN_AGE:-7}"
|
DOWNLOAD_ORPHAN_AGE="${DOWNLOAD_ORPHAN_AGE:-7}"
|
||||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB="${DOWNLOAD_ORPHAN_MIN_VIDEO_MB:-50}"
|
DOWNLOAD_ORPHAN_MIN_VIDEO_MB="${DOWNLOAD_ORPHAN_MIN_VIDEO_MB:-50}"
|
||||||
|
DOWNLOAD_ORPHAN_MIN_AUDIO_MB="${DOWNLOAD_ORPHAN_MIN_AUDIO_MB:-2}"
|
||||||
|
DOWNLOAD_ORPHAN_KEEP_MARKER="${DOWNLOAD_ORPHAN_KEEP_MARKER:-.vv-keep}"
|
||||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB="${DOWNLOAD_ORPHAN_MAX_DELETE_GB:-100}"
|
DOWNLOAD_ORPHAN_MAX_DELETE_GB="${DOWNLOAD_ORPHAN_MAX_DELETE_GB:-100}"
|
||||||
|
|
||||||
if [[ "${SHOW_STATUS:-false}" == true ]]; then
|
if [[ "${SHOW_STATUS:-false}" == true ]]; then
|
||||||
@@ -204,9 +217,10 @@ if [[ "${SHOW_STATUS:-false}" == true ]]; then
|
|||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
echo "$ICON_GEAR Enabled: ${DOWNLOAD_ORPHAN_CLEANER_ENABLED}"
|
echo "$ICON_GEAR Enabled: ${DOWNLOAD_ORPHAN_CLEANER_ENABLED}"
|
||||||
echo "$ICON_TIME Age gate: ${DOWNLOAD_ORPHAN_AGE}d"
|
echo "$ICON_TIME Age gate: ${DOWNLOAD_ORPHAN_AGE}d"
|
||||||
echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M"
|
echo "$ICON_DISK Junk threshold: ${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}M video / ${DOWNLOAD_ORPHAN_MIN_AUDIO_MB}M audio"
|
||||||
echo "$ICON_SHIELD Delete cap: ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G"
|
echo "$ICON_SHIELD Delete cap: ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G"
|
||||||
for arr in SONARR RADARR; do
|
echo "$ICON_SHIELD Keep marker: ${DOWNLOAD_ORPHAN_KEEP_MARKER}"
|
||||||
|
for arr in SONARR RADARR LIDARR; do
|
||||||
dir_var="${MY_ID}_${arr}_DOWNLOAD_DIR"
|
dir_var="${MY_ID}_${arr}_DOWNLOAD_DIR"
|
||||||
echo "$ICON_CLEAN ${arr}: ${!dir_var:-<not configured>}"
|
echo "$ICON_CLEAN ${arr}: ${!dir_var:-<not configured>}"
|
||||||
done
|
done
|
||||||
@@ -221,13 +235,16 @@ AGE_CUTOFF=$(( $(date +%s) - DOWNLOAD_ORPHAN_AGE * 86400 ))
|
|||||||
TOTAL_DELETED=0
|
TOTAL_DELETED=0
|
||||||
TOTAL_DELETED_MB=0
|
TOTAL_DELETED_MB=0
|
||||||
TOTAL_HELD=0
|
TOTAL_HELD=0
|
||||||
|
TOTAL_DEFERRED=0
|
||||||
|
TOTAL_KEPT=0
|
||||||
TOTAL_SCANS=0
|
TOTAL_SCANS=0
|
||||||
|
|
||||||
echo "━━━━━ $ICON_CLEAN DOWNLOAD ORPHAN CLEANER ━━━━━"
|
echo "━━━━━ $ICON_CLEAN DOWNLOAD ORPHAN CLEANER ━━━━━"
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
[[ "$DRY_RUN" == true ]] && echo "$ICON_SKIP DRY RUN — nothing will be deleted or imported"
|
[[ "$DRY_RUN" == true ]] && echo "$ICON_SKIP DRY RUN — nothing will be deleted or imported"
|
||||||
|
|
||||||
for arr in sonarr radarr; do
|
for arr in sonarr radarr lidarr; do
|
||||||
|
api_ver="v3"; [[ "$arr" == "lidarr" ]] && api_ver="v1"
|
||||||
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
url_var="${arr^^}_URL"; key_var="${arr^^}_API_KEY"
|
||||||
arr_url="${!url_var:-}"; arr_key="${!key_var:-}"
|
arr_url="${!url_var:-}"; arr_key="${!key_var:-}"
|
||||||
dir_var="${MY_ID}_${arr^^}_DOWNLOAD_DIR"
|
dir_var="${MY_ID}_${arr^^}_DOWNLOAD_DIR"
|
||||||
@@ -247,24 +264,38 @@ for arr in sonarr radarr; do
|
|||||||
echo "━━━ $ICON_SYNC ${arr^} — $dl_dir ━━━"
|
echo "━━━ $ICON_SYNC ${arr^} — $dl_dir ━━━"
|
||||||
|
|
||||||
ver_var="${arr^^}_VERSION_MAJOR"
|
ver_var="${arr^^}_VERSION_MAJOR"
|
||||||
check_arr_version "$arr_url" "$arr_key" "v3" "${!ver_var}" "${arr^}" || {
|
check_arr_version "$arr_url" "$arr_key" "$api_ver" "${!ver_var}" "${arr^}" || {
|
||||||
warn "${arr^} version check failed — skipping this arr"
|
warn "${arr^} version check failed — skipping this arr"
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
# min_mb is per-arr because the JUNK test is "contains no real media file". A 50MB floor
|
||||||
|
# is right for video and catastrophic for audio — most single tracks never reach it, so
|
||||||
|
# every music folder would classify as JUNK and be deleted regardless of import state.
|
||||||
|
case "$arr" in
|
||||||
|
sonarr)
|
||||||
|
queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true"
|
||||||
|
exts_var="SONARR_EXTENSIONS"
|
||||||
|
scan_command="DownloadedEpisodesScan"
|
||||||
|
library_endpoint="series"
|
||||||
|
min_mb="$DOWNLOAD_ORPHAN_MIN_VIDEO_MB"
|
||||||
|
;;
|
||||||
|
radarr)
|
||||||
|
queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true"
|
||||||
|
exts_var="RADARR_EXTENSIONS"
|
||||||
|
scan_command="DownloadedMoviesScan"
|
||||||
|
library_endpoint="movie"
|
||||||
|
min_mb="$DOWNLOAD_ORPHAN_MIN_VIDEO_MB"
|
||||||
|
;;
|
||||||
|
lidarr)
|
||||||
|
queue_endpoint="queue?pageSize=1000&includeUnknownArtistItems=true"
|
||||||
|
exts_var="LIDARR_EXTENSIONS"
|
||||||
|
scan_command="DownloadedAlbumsScan"
|
||||||
|
library_endpoint="artist"
|
||||||
|
min_mb="$DOWNLOAD_ORPHAN_MIN_AUDIO_MB"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [[ "$arr" == "sonarr" ]]; then
|
QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "$api_ver" "$queue_endpoint" "${arr^}") || {
|
||||||
queue_endpoint="queue?pageSize=1000&includeUnknownSeriesItems=true"
|
|
||||||
exts_var="SONARR_EXTENSIONS"
|
|
||||||
scan_command="DownloadedEpisodesScan"
|
|
||||||
library_endpoint="series"
|
|
||||||
else
|
|
||||||
queue_endpoint="queue?pageSize=1000&includeUnknownMovieItems=true"
|
|
||||||
exts_var="RADARR_EXTENSIONS"
|
|
||||||
scan_command="DownloadedMoviesScan"
|
|
||||||
library_endpoint="movie"
|
|
||||||
fi
|
|
||||||
|
|
||||||
QUEUE_JSON=$(arr_api "$arr_url" "$arr_key" "v3" "$queue_endpoint" "${arr^}") || {
|
|
||||||
error "${arr^} queue fetch failed — cannot tell tracked from orphaned, skipping this arr"
|
error "${arr^} queue fetch failed — cannot tell tracked from orphaned, skipping this arr"
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -282,11 +313,21 @@ for arr in sonarr radarr; do
|
|||||||
SCAN_PATHS=()
|
SCAN_PATHS=()
|
||||||
UNMATCHED_PATHS=()
|
UNMATCHED_PATHS=()
|
||||||
UNMATCHED_SIZES=()
|
UNMATCHED_SIZES=()
|
||||||
arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0
|
arr_tracked=0; arr_recent=0; arr_held=0; arr_delete_mb=0; arr_kept=0
|
||||||
|
|
||||||
while IFS= read -r entry; do
|
while IFS= read -r entry; do
|
||||||
base="${entry##*/}"
|
base="${entry##*/}"
|
||||||
|
|
||||||
|
# An operator keep-marker outranks every verdict below. Needed because REDUNDANT only
|
||||||
|
# asks "does the library hold this album", not "at what quality" — a lossless rip whose
|
||||||
|
# library copy is MP3 is redundant by that test and would be swept on the next run.
|
||||||
|
# The marker is a file inside the folder rather than a conf list so it survives renames
|
||||||
|
# and cannot drift out of sync with what is actually on disk.
|
||||||
|
if [[ -e "$entry/$DOWNLOAD_ORPHAN_KEEP_MARKER" ]]; then
|
||||||
|
arr_kept=$((arr_kept + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ -n "${PROTECTED[$base]:-}" ]]; then
|
if [[ -n "${PROTECTED[$base]:-}" ]]; then
|
||||||
arr_tracked=$((arr_tracked + 1))
|
arr_tracked=$((arr_tracked + 1))
|
||||||
continue
|
continue
|
||||||
@@ -298,18 +339,30 @@ for arr in sonarr radarr; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
has_video=false
|
# JUNK means "holds no real media". That verdict is only as good as the extension
|
||||||
|
# list, and a missing extension turns real content into a delete — 2026-08-21 the
|
||||||
|
# audio list had no "wv", which classified 23 folders of WavPack lossless (1.5G per
|
||||||
|
# file) as junk. So a folder with large files that are merely *unrecognised* is held
|
||||||
|
# for review, never deleted; only a folder with nothing big in it at all is junk.
|
||||||
|
has_media=false
|
||||||
|
big_unknown=0
|
||||||
while IFS= read -r f; do
|
while IFS= read -r f; do
|
||||||
if has_extension "$f" "${arr_exts[@]}"; then
|
if has_extension "$f" "${arr_exts[@]}"; then
|
||||||
has_video=true
|
has_media=true
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
done < <(find "$entry" -type f -size +"${DOWNLOAD_ORPHAN_MIN_VIDEO_MB}"M 2>/dev/null)
|
big_unknown=$((big_unknown + 1))
|
||||||
|
done < <(find "$entry" -type f -size +"${min_mb}"M 2>/dev/null)
|
||||||
|
|
||||||
size_mb=$(du -sm "$entry" 2>/dev/null | cut -f1)
|
size_mb=$(dir_size_mb "$entry") || size_mb=0
|
||||||
size_mb=${size_mb:-0}
|
|
||||||
|
|
||||||
if [[ "$has_video" == false ]]; then
|
if [[ "$has_media" == false ]] && (( big_unknown > 0 )); then
|
||||||
|
warn " no recognised media, but $big_unknown large file(s) of unknown type — holding: $base"
|
||||||
|
arr_held=$((arr_held + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$has_media" == false ]]; then
|
||||||
DELETE_PATHS+=("$entry")
|
DELETE_PATHS+=("$entry")
|
||||||
DELETE_SIZES+=("$size_mb")
|
DELETE_SIZES+=("$size_mb")
|
||||||
DELETE_LABELS+=("JUNK")
|
DELETE_LABELS+=("JUNK")
|
||||||
@@ -318,20 +371,43 @@ for arr in sonarr radarr; do
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
enc_title=$(jq -rn --arg t "$base" '$t|@uri')
|
enc_title=$(jq -rn --arg t "$base" '$t|@uri')
|
||||||
parse=$(arr_api "$arr_url" "$arr_key" "v3" "parse?title=${enc_title}" "${arr^}") || {
|
parse=$(arr_api "$arr_url" "$arr_key" "$api_ver" "parse?title=${enc_title}" "${arr^}") || {
|
||||||
warn " parse failed for: $base — holding"
|
warn " parse failed for: $base — holding"
|
||||||
arr_held=$((arr_held + 1))
|
arr_held=$((arr_held + 1))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if [[ "$arr" == "sonarr" ]]; then
|
case "$arr" in
|
||||||
matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)')
|
sonarr)
|
||||||
missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length')
|
matched=$(echo "$parse" | jq '(.series != null) and ((.episodes | length) > 0)')
|
||||||
else
|
missing=$(echo "$parse" | jq '[.episodes[]? | select(.hasFile == false)] | length')
|
||||||
# Radarr's parse never populates hasFile — movieFileId is the reliable signal
|
;;
|
||||||
matched=$(echo "$parse" | jq '.movie != null')
|
radarr)
|
||||||
missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end')
|
# Radarr's parse never populates hasFile — movieFileId is the reliable signal
|
||||||
fi
|
matched=$(echo "$parse" | jq '.movie != null')
|
||||||
|
missing=$(echo "$parse" | jq 'if (.movie.movieFileId // 0) > 0 then 0 else 1 end')
|
||||||
|
;;
|
||||||
|
lidarr)
|
||||||
|
# Lidarr's parse returns albums with statistics:null, so the track count has
|
||||||
|
# to be read back from album/{id} — the same shape of gap as Radarr's hasFile.
|
||||||
|
matched=$(echo "$parse" | jq '(.artist != null) and ((.albums | length) > 0)')
|
||||||
|
missing=1
|
||||||
|
if [[ "$matched" == true ]]; then
|
||||||
|
album_id=$(echo "$parse" | jq -r '.albums[0].id // empty')
|
||||||
|
if [[ -z "$album_id" ]]; then
|
||||||
|
warn " parse matched but returned no album id: $base — holding"
|
||||||
|
arr_held=$((arr_held + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
album_json=$(arr_api "$arr_url" "$arr_key" "$api_ver" "album/$album_id" "${arr^}") || {
|
||||||
|
warn " album lookup failed for: $base — holding"
|
||||||
|
arr_held=$((arr_held + 1))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
missing=$(echo "$album_json" | jq 'if ((.statistics.trackFileCount // 0) > 0) then 0 else 1 end')
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
if [[ "$matched" != true ]]; then
|
if [[ "$matched" != true ]]; then
|
||||||
UNMATCHED_PATHS+=("$entry")
|
UNMATCHED_PATHS+=("$entry")
|
||||||
@@ -354,7 +430,7 @@ for arr in sonarr radarr; do
|
|||||||
# fails on a small batch that is legitimately all-unmatched, which is the normal case
|
# fails on a small batch that is legitimately all-unmatched, which is the normal case
|
||||||
# once daily runs have caught up.
|
# once daily runs have caught up.
|
||||||
if (( ${#UNMATCHED_PATHS[@]} > 0 )); then
|
if (( ${#UNMATCHED_PATHS[@]} > 0 )); then
|
||||||
library_count=$(arr_api "$arr_url" "$arr_key" "v3" "$library_endpoint" "${arr^}" | jq 'length' 2>/dev/null)
|
library_count=$(arr_api "$arr_url" "$arr_key" "$api_ver" "$library_endpoint" "${arr^}" | jq 'length' 2>/dev/null)
|
||||||
if [[ ! "$library_count" =~ ^[0-9]+$ ]] || (( library_count == 0 )); then
|
if [[ ! "$library_count" =~ ^[0-9]+$ ]] || (( library_count == 0 )); then
|
||||||
warn " ${arr^}: library reports ${library_count:-no} titles — cannot trust 'no match', holding ${#UNMATCHED_PATHS[@]} unmatched"
|
warn " ${arr^}: library reports ${library_count:-no} titles — cannot trust 'no match', holding ${#UNMATCHED_PATHS[@]} unmatched"
|
||||||
arr_held=$((arr_held + ${#UNMATCHED_PATHS[@]}))
|
arr_held=$((arr_held + ${#UNMATCHED_PATHS[@]}))
|
||||||
@@ -368,26 +444,59 @@ for arr in sonarr radarr; do
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if (( arr_delete_mb / 1024 > DOWNLOAD_ORPHAN_MAX_DELETE_GB )) && [[ "$I_KNOW" != true ]]; then
|
# The cap is a per-run risk budget, not a reason to do nothing. Aborting the whole pass
|
||||||
error "${arr^}: delete total $((arr_delete_mb / 1024))G exceeds cap of ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G — aborting delete pass"
|
# once the backlog exceeds it is self-defeating: the backlog can never shrink below the
|
||||||
notify "${arr^} download orphan delete total $((arr_delete_mb / 1024))G exceeds ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap on $(hostname) — possible partial queue data, nothing deleted. Re-run with --i-know-what-im-doing if legitimate." \
|
# cap on its own, so every later run aborts too and the pool fills anyway (exactly how
|
||||||
|
# 347G accumulated here by 2026-08-21). Delete in ascending order of risk instead, stop
|
||||||
|
# at the cap, and defer the rest to the next run so a backlog drains over days.
|
||||||
|
#
|
||||||
|
# Live downloads are already protected by DOWNLOAD_ORPHAN_AGE, not by this cap — anything
|
||||||
|
# in flight is younger than the age gate and never reaches classification. That is what
|
||||||
|
# makes draining safe: the partial-queue-data case the cap was written for cannot put a
|
||||||
|
# still-downloading entry in these arrays.
|
||||||
|
cap_mb=$((DOWNLOAD_ORPHAN_MAX_DELETE_GB * 1024))
|
||||||
|
cap_active=true
|
||||||
|
[[ "$I_KNOW" == true || "$DRY_RUN" == true ]] && cap_active=false
|
||||||
|
|
||||||
|
arr_deferred=0; arr_deferred_mb=0; arr_run_mb=0
|
||||||
|
|
||||||
|
if [[ "$cap_active" == true ]] && (( arr_delete_mb > cap_mb )); then
|
||||||
|
warn " ${arr^}: $((arr_delete_mb / 1024))G classified vs ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G cap — deleting safest-first up to the cap, deferring the rest"
|
||||||
|
notify "${arr^} download orphan backlog is $((arr_delete_mb / 1024))G on $(hostname), above the ${DOWNLOAD_ORPHAN_MAX_DELETE_GB}G per-run cap. Draining safest-first; the remainder follows on later runs. Re-run with --i-know-what-im-doing to clear it in one pass." \
|
||||||
"Download Orphan Cleaner" "warning"
|
"Download Orphan Cleaner" "warning"
|
||||||
unset PROTECTED
|
|
||||||
continue
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
for i in "${!DELETE_PATHS[@]}"; do
|
# JUNK first (no media at all), then REDUNDANT (parse-verified already in the library),
|
||||||
entry="${DELETE_PATHS[$i]}"
|
# then UNMATCHED last — it rests on "the arr does not know this title", the weakest of
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# the three signals, so it is the first thing the cap defers.
|
||||||
echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
for pass in JUNK REDUNDANT UNMATCHED; do
|
||||||
else
|
for i in "${!DELETE_PATHS[@]}"; do
|
||||||
echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
[[ "${DELETE_LABELS[$i]}" == "$pass" ]] || continue
|
||||||
rm -rf "$entry"
|
entry="${DELETE_PATHS[$i]}"
|
||||||
fi
|
|
||||||
TOTAL_DELETED=$((TOTAL_DELETED + 1))
|
if [[ "$cap_active" == true ]] && (( arr_run_mb + DELETE_SIZES[i] > cap_mb )); then
|
||||||
TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i]))
|
arr_deferred=$((arr_deferred + 1))
|
||||||
|
arr_deferred_mb=$((arr_deferred_mb + DELETE_SIZES[i]))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo " $ICON_SKIP would delete [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
||||||
|
else
|
||||||
|
echo " $ICON_TRASH deleting [${DELETE_LABELS[$i]}]: ${entry##*/} (${DELETE_SIZES[$i]}M)"
|
||||||
|
rm -rf "$entry"
|
||||||
|
fi
|
||||||
|
arr_run_mb=$((arr_run_mb + DELETE_SIZES[i]))
|
||||||
|
TOTAL_DELETED=$((TOTAL_DELETED + 1))
|
||||||
|
TOTAL_DELETED_MB=$((TOTAL_DELETED_MB + DELETE_SIZES[i]))
|
||||||
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if (( arr_deferred > 0 )); then
|
||||||
|
echo " $ICON_WARN ${arr^}: deferred $arr_deferred entries ($((arr_deferred_mb / 1024))G) to the next run — cap reached"
|
||||||
|
TOTAL_DEFERRED=$((TOTAL_DEFERRED + arr_deferred))
|
||||||
|
fi
|
||||||
|
|
||||||
for base in "${SCAN_PATHS[@]}"; do
|
for base in "${SCAN_PATHS[@]}"; do
|
||||||
if [[ -z "$container_dir" ]]; then
|
if [[ -z "$container_dir" ]]; then
|
||||||
echo " $ICON_WARN IMPORTABLE but ${cdir_var} not set — holding: $base"
|
echo " $ICON_WARN IMPORTABLE but ${cdir_var} not set — holding: $base"
|
||||||
@@ -406,8 +515,9 @@ for arr in sonarr radarr; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_recent recent, ${#DELETE_PATHS[@]} deleted ($((arr_delete_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held"
|
echo " $ICON_SUMMARY ${arr^}: $arr_tracked tracked, $arr_kept kept, $arr_recent recent, $((${#DELETE_PATHS[@]} - arr_deferred)) deleted ($((arr_run_mb / 1024))G), $arr_deferred deferred ($((arr_deferred_mb / 1024))G), ${#SCAN_PATHS[@]} import scans, $arr_held held"
|
||||||
TOTAL_HELD=$((TOTAL_HELD + arr_held))
|
TOTAL_HELD=$((TOTAL_HELD + arr_held))
|
||||||
|
TOTAL_KEPT=$((TOTAL_KEPT + arr_kept))
|
||||||
unset PROTECTED
|
unset PROTECTED
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -416,6 +526,8 @@ echo "━━━━━ $ICON_DONE SUMMARY ━━━━━"
|
|||||||
echo "$ICON_TRASH Deleted: $TOTAL_DELETED ($((TOTAL_DELETED_MB / 1024))G)"
|
echo "$ICON_TRASH Deleted: $TOTAL_DELETED ($((TOTAL_DELETED_MB / 1024))G)"
|
||||||
echo "$ICON_RUN Import scans: $TOTAL_SCANS"
|
echo "$ICON_RUN Import scans: $TOTAL_SCANS"
|
||||||
echo "$ICON_WARN Held: $TOTAL_HELD"
|
echo "$ICON_WARN Held: $TOTAL_HELD"
|
||||||
|
echo "$ICON_SKIP Deferred: $TOTAL_DEFERRED"
|
||||||
|
echo "$ICON_SHIELD Kept (marker): $TOTAL_KEPT"
|
||||||
|
|
||||||
# Held alone never notifies — there is always something awaiting review, and on a daily
|
# Held alone never notifies — there is always something awaiting review, and on a daily
|
||||||
# schedule that would be a notification every morning saying nothing happened.
|
# schedule that would be a notification every morning saying nothing happened.
|
||||||
|
|||||||
@@ -44,7 +44,7 @@
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
# Remote API Access — Cache-First, SSH Fallback
|
# Remote API Access — Cache-First, SSH Fallback
|
||||||
# If conf_sync.sh has populated /tmp/.cache/vv/d/ and
|
# If conf_sync.sh has populated /tmp/varaverk/conf/ and
|
||||||
# load_config.sh has sourced it, HOST*_<ARR>_API_KEY vars are available
|
# load_config.sh has sourced it, HOST*_<ARR>_API_KEY vars are available
|
||||||
# in the environment. Remote functions use them to call the arr API
|
# in the environment. Remote functions use them to call the arr API
|
||||||
# directly over Tailscale (no SSH, no remote shell). If the cached key
|
# directly over Tailscale (no SSH, no remote shell). If the cached key
|
||||||
|
|||||||
Executable → Regular
+70
-8
@@ -457,6 +457,30 @@ NOW=$(date +%s)
|
|||||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||||
> "$TO_DELETE_FILE"
|
> "$TO_DELETE_FILE"
|
||||||
|
|
||||||
|
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||||
|
# LIDARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||||
|
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||||
|
# rebuilt from each run rather than edited, which is what prunes it.
|
||||||
|
LIDARR_ORPHAN_STRIKE_LIMIT="${LIDARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||||
|
STRIKES_FILE="${LIDARR_ORPHAN_STRIKES_FILE:-$DB_DIR/lidarr_orphan_strikes.tsv}"
|
||||||
|
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||||
|
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||||
|
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||||
|
> "$STRIKES_NEW"
|
||||||
|
HELD_COUNT=0
|
||||||
|
HELD_BYTES=0
|
||||||
|
|
||||||
|
orphan_strike_ok() {
|
||||||
|
local path="$1" prev strikes
|
||||||
|
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||||
|
strikes=$(( ${prev:-0} + 1 ))
|
||||||
|
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||||
|
(( strikes >= LIDARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||||
|
warn " strike $strikes/$LIDARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||||
@@ -493,12 +517,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
|||||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||||
(( ORPHAN_COUNT++ ))
|
(( ORPHAN_COUNT++ ))
|
||||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||||
|
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||||
else
|
else
|
||||||
log "JUNK: $filepath"
|
log "JUNK: $filepath"
|
||||||
(( JUNK_COUNT++ ))
|
(( JUNK_COUNT++ ))
|
||||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||||
|
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||||
@@ -506,13 +532,43 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
|||||||
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
# 4.3ms), since find already has to stat() every entry anyway to know it's -type f.
|
||||||
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
|
done < <(find "$LIDARR_MUSIC_ROOT" -type f -printf '%s %C@ %p\n' 2>/dev/null)
|
||||||
|
|
||||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
# run, so it must not appear in the denominator the budget reports against.
|
||||||
|
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||||
|
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||||
|
|
||||||
|
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||||
|
# the next real run delete a run early.
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$LIDARR_MAX_DELETE_GB" "Lidarr Cleanup"
|
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||||
|
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||||
|
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||||
|
|
||||||
|
if [[ "$I_KNOW" == true ]]; then
|
||||||
|
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||||
|
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||||
|
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||||
|
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||||
|
else
|
||||||
|
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$LIDARR_MAX_DELETE_GB"
|
||||||
|
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||||
|
error "Single file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||||
|
error " $_BUDGET_STUCK"
|
||||||
|
error "Raise LIDARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||||
|
notify "Lidarr cleanup stalled on $(hostname) — one file exceeds the ${LIDARR_MAX_DELETE_GB}GB budget" \
|
||||||
|
"Lidarr Cleanup" "warning"
|
||||||
|
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||||
|
warn "Budget ${LIDARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||||
|
notify "Lidarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||||
|
"Lidarr Cleanup" "normal"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||||
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
|
# All safety layers passed — delete orphans and junk. Reuses TO_DELETE_FILE from the
|
||||||
@@ -521,7 +577,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||||
done < "$TO_DELETE_FILE"
|
done < "$BUDGET_FILE"
|
||||||
|
|
||||||
info "Cleaning up empty folders..."
|
info "Cleaning up empty folders..."
|
||||||
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
find "$LIDARR_MUSIC_ROOT" -mindepth 1 -type d -empty -delete 2>/dev/null
|
||||||
@@ -546,6 +602,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (cover art, metadata
|
|||||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
|
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${LIDARR_ORPHAN_AGE} days)"
|
||||||
|
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||||
|
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${LIDARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||||
|
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||||
|
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${LIDARR_MAX_DELETE_GB}GB run budget"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
@@ -554,8 +614,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
|||||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Clean — nothing to remove"
|
echo "$ICON_DONE Clean — nothing to remove"
|
||||||
else
|
else
|
||||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||||
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||||
|
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||||
|
notify "Lidarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "warning"
|
||||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||||
notify_emby_scan
|
notify_emby_scan
|
||||||
fi
|
fi
|
||||||
|
|||||||
Executable → Regular
+134
-8
@@ -450,9 +450,46 @@ NOW=$(date +%s)
|
|||||||
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
|
# just delete them directly instead of re-walking and re-classifying every SCAN_ROOTS entry a
|
||||||
# second time (2026-07-17) — the size-threshold check below needs to know the total before
|
# second time (2026-07-17) — the size-threshold check below needs to know the total before
|
||||||
# deleting anything, not before knowing what to delete.
|
# deleting anything, not before knowing what to delete.
|
||||||
|
# Carries size and ctime alongside the path now, because the budget pass below has to order by
|
||||||
|
# age and stop at a byte ceiling — neither of which a bare path list can answer.
|
||||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||||
> "$TO_DELETE_FILE"
|
> "$TO_DELETE_FILE"
|
||||||
|
|
||||||
|
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# A file must classify for deletion on RADARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is
|
||||||
|
# actually removed. Gate 6 already refuses a run whose tracked count collapsed; this covers the
|
||||||
|
# partial failure underneath that threshold — one root folder failing to enumerate makes its
|
||||||
|
# movies look orphaned while the overall percentage still looks fine, and a transient fault will
|
||||||
|
# not reproduce on the next run.
|
||||||
|
#
|
||||||
|
# The file is REBUILT from this run's classifications rather than edited in place, which is what
|
||||||
|
# prunes it: anything that stopped being an orphan simply is not written again, so a file that
|
||||||
|
# Radarr re-adopts loses its strikes without needing a reset pass to find it.
|
||||||
|
#
|
||||||
|
# Keyed by host path, which is why this could not have worked before 2026-08-26 — wd_state_set
|
||||||
|
# built a regex from the key, and a release tag like [Bluray-1080p] holds the reversed range 1-0,
|
||||||
|
# so every write truncated the store to one line. See common.sh.
|
||||||
|
RADARR_ORPHAN_STRIKE_LIMIT="${RADARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||||
|
STRIKES_FILE="${RADARR_ORPHAN_STRIKES_FILE:-$DB_DIR/radarr_orphan_strikes.tsv}"
|
||||||
|
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||||
|
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||||
|
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||||
|
> "$STRIKES_NEW"
|
||||||
|
HELD_COUNT=0
|
||||||
|
HELD_BYTES=0
|
||||||
|
|
||||||
|
# Records this run's strike for a file and says whether it has served enough of them.
|
||||||
|
# Returns 0 when the file may be deleted, 1 when it is still accruing.
|
||||||
|
orphan_strike_ok() {
|
||||||
|
local path="$1" prev strikes
|
||||||
|
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||||
|
strikes=$(( ${prev:-0} + 1 ))
|
||||||
|
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||||
|
(( strikes >= RADARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||||
|
warn " strike $strikes/$RADARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||||
@@ -487,12 +524,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
|||||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||||
(( ORPHAN_COUNT++ ))
|
(( ORPHAN_COUNT++ ))
|
||||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||||
|
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||||
else
|
else
|
||||||
log "JUNK: $filepath"
|
log "JUNK: $filepath"
|
||||||
(( JUNK_COUNT++ ))
|
(( JUNK_COUNT++ ))
|
||||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||||
|
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||||
@@ -504,13 +543,94 @@ done < <(
|
|||||||
done | sort -u
|
done | sort -u
|
||||||
)
|
)
|
||||||
|
|
||||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
# Eligible, not classified. A file still serving its strikes was counted as an orphan above — it
|
||||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
# is one — but it is not going to be deleted this run, so it must not appear in the denominator
|
||||||
|
# the budget reports against or the run claims to have skipped work it never queued.
|
||||||
|
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||||
|
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||||
|
|
||||||
|
# Rebuilt, never edited: a path absent from this run is absent from the file, so a file Radarr
|
||||||
|
# re-adopts drops its strikes with no reset pass needed. Skipped on a dry run — a preview that
|
||||||
|
# advanced real strike counters would make the next real run delete a run early.
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$RADARR_MAX_DELETE_GB" "Radarr Cleanup"
|
# The ceiling is a per-run budget, not a veto. It still means what it always meant — no single run
|
||||||
|
# removes more than RADARR_MAX_DELETE_GB — but a backlog larger than the ceiling now drains over
|
||||||
|
# consecutive nights instead of failing the orchestrator forever on a queue it cannot clear.
|
||||||
|
# ── AI note (AI_ASSIST_CLEANUP) ───────────────────────────────────────────────────────────────
|
||||||
|
# Describes the shape of what was classified. It decides nothing: the eligible set, the budget and
|
||||||
|
# the strikes are all settled above and none of them read this. Switch AI_ASSIST_CLEANUP off and
|
||||||
|
# the run removes exactly the same files — the log just loses a paragraph.
|
||||||
|
#
|
||||||
|
# ctime clustering is the signal worth surfacing. A normal upgrade cycle dribbles in over weeks; a
|
||||||
|
# lump sharing one narrow ctime window with mtimes spread across months is a bulk write-back, which
|
||||||
|
# is what a partnership merge against a partner holding older copies produces. That distinction
|
||||||
|
# took a person an evening on 2026-08-26 and is the whole reason this note exists.
|
||||||
|
if [[ "$ORPHAN_COUNT" -gt 0 ]] && [[ -s "$TO_DELETE_FILE" ]]; then
|
||||||
|
_ai_ev=$(awk -F'\t' '
|
||||||
|
{ n++; bytes += $1
|
||||||
|
c = int($2)
|
||||||
|
if (cmin == 0 || c < cmin) cmin = c
|
||||||
|
if (c > cmax) cmax = c
|
||||||
|
bucket[int(c / 21600)]++ }
|
||||||
|
END {
|
||||||
|
for (b in bucket) if (bucket[b] > top) { top = bucket[b] }
|
||||||
|
printf "files=%d bytes_gb=%.1f ctime_span_hours=%.1f largest_6h_ctime_bucket=%d\n",
|
||||||
|
n, bytes/1073741824, (cmax-cmin)/3600, top
|
||||||
|
}' "$TO_DELETE_FILE")
|
||||||
|
_ai_mt=$(cut -d"$(printf '\t')" -f3 "$TO_DELETE_FILE" | head -8 \
|
||||||
|
| while IFS= read -r p; do [[ -f "$p" ]] && \
|
||||||
|
printf '%s %s\n' "$(stat -c %y "$p" 2>/dev/null | cut -c1-7)" "$(basename "$p")"; done)
|
||||||
|
|
||||||
|
_ai_note=$(ai_assist_note AI_ASSIST_CLEANUP "You are looking at files an automated media-library cleanup has classified for deletion on an Unraid server. They are files on disk that the Radarr database no longer references.
|
||||||
|
|
||||||
|
EVIDENCE
|
||||||
|
$_ai_ev
|
||||||
|
sample (modification month, then path):
|
||||||
|
$_ai_mt
|
||||||
|
|
||||||
|
A normal quality-upgrade cycle produces orphans whose ctimes are spread out over weeks, because each upgrade happens on its own day. A bulk event - a sync or restore writing files back onto this host - produces orphans sharing one narrow ctime window while their modification times stay spread across months, because the copy preserves modification time but resets ctime.
|
||||||
|
|
||||||
|
In no more than three sentences, say which of those two this looks like and name the numbers above that support it. Do not recommend an action. Do not speculate beyond the evidence given.") || _ai_note=""
|
||||||
|
|
||||||
|
if [[ -n "$_ai_note" ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR AI note on this classification ━━━"
|
||||||
|
printf '%s\n' "$_ai_note"
|
||||||
|
fi
|
||||||
|
unset _ai_ev _ai_mt
|
||||||
|
fi
|
||||||
|
|
||||||
|
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||||
|
|
||||||
|
if [[ "$I_KNOW" == true ]]; then
|
||||||
|
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||||
|
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||||
|
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||||
|
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||||
|
else
|
||||||
|
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$RADARR_MAX_DELETE_GB"
|
||||||
|
|
||||||
|
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||||
|
# One file larger than the whole budget can never fit, so it would be re-found and
|
||||||
|
# re-deferred every night. Name it rather than loop on it silently.
|
||||||
|
error "Single file exceeds the ${RADARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||||
|
error " $_BUDGET_STUCK"
|
||||||
|
error "Raise RADARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||||
|
notify "Radarr cleanup stalled on $(hostname) — one file exceeds the ${RADARR_MAX_DELETE_GB}GB budget" \
|
||||||
|
"Radarr Cleanup" "warning"
|
||||||
|
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||||
|
warn "Budget ${RADARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||||
|
warn "Oldest first — the deferred files are the newest and are re-evaluated tomorrow"
|
||||||
|
notify "Radarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred to the next run" \
|
||||||
|
"Radarr Cleanup" "normal"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||||
@@ -519,7 +639,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||||
done < "$TO_DELETE_FILE"
|
done < "$BUDGET_FILE"
|
||||||
|
|
||||||
info "Cleaning up empty folders..."
|
info "Cleaning up empty folders..."
|
||||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||||
@@ -545,6 +665,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
|||||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${RADARR_ORPHAN_AGE} days)"
|
||||||
|
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||||
|
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${RADARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||||
|
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||||
|
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${RADARR_MAX_DELETE_GB}GB run budget"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
@@ -553,8 +677,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
|||||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Clean — nothing to remove"
|
echo "$ICON_DONE Clean — nothing to remove"
|
||||||
else
|
else
|
||||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
# What was actually removed, not what was classified. With a budget in force those differ, and
|
||||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
# reporting the classification as the outcome is the oldest bug shape in this codebase.
|
||||||
|
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||||
|
notify "Radarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED classified files ($(format_bytes "$_BUDGET_KEPT_BYTES"))$([[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && echo ", $_BUDGET_DEFERRED_COUNT deferred")" \
|
||||||
"Radarr Cleanup" "warning"
|
"Radarr Cleanup" "warning"
|
||||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||||
notify_emby_scan
|
notify_emby_scan
|
||||||
|
|||||||
Executable → Regular
+70
-8
@@ -440,6 +440,30 @@ NOW=$(date +%s)
|
|||||||
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
TO_DELETE_FILE="$TMP_DIR/to_delete_paths.txt"
|
||||||
> "$TO_DELETE_FILE"
|
> "$TO_DELETE_FILE"
|
||||||
|
|
||||||
|
# ── Orphan strikes ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Same contract as radarr_cleanup.sh: a file must classify for deletion on
|
||||||
|
# SONARR_ORPHAN_STRIKE_LIMIT consecutive runs before it is removed. Covers the partial
|
||||||
|
# classification failure that is too small to trip the tracked-count floor above. The file is
|
||||||
|
# rebuilt from each run rather than edited, which is what prunes it.
|
||||||
|
SONARR_ORPHAN_STRIKE_LIMIT="${SONARR_ORPHAN_STRIKE_LIMIT:-2}"
|
||||||
|
STRIKES_FILE="${SONARR_ORPHAN_STRIKES_FILE:-$DB_DIR/sonarr_orphan_strikes.tsv}"
|
||||||
|
mkdir -p "$(dirname "$STRIKES_FILE")" 2>/dev/null || true
|
||||||
|
touch "$STRIKES_FILE" 2>/dev/null || true
|
||||||
|
STRIKES_NEW="$TMP_DIR/strikes_new.tsv"
|
||||||
|
> "$STRIKES_NEW"
|
||||||
|
HELD_COUNT=0
|
||||||
|
HELD_BYTES=0
|
||||||
|
|
||||||
|
orphan_strike_ok() {
|
||||||
|
local path="$1" prev strikes
|
||||||
|
prev=$(wd_state_get "$path" "$STRIKES_FILE"); prev="${prev//[^0-9]/}"
|
||||||
|
strikes=$(( ${prev:-0} + 1 ))
|
||||||
|
printf '%s:%s\n' "$path" "$strikes" >> "$STRIKES_NEW"
|
||||||
|
(( strikes >= SONARR_ORPHAN_STRIKE_LIMIT )) && return 0
|
||||||
|
warn " strike $strikes/$SONARR_ORPHAN_STRIKE_LIMIT — not removing yet: $path"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
while read -r FILE_SIZE FILE_CTIME filepath; do
|
while read -r FILE_SIZE FILE_CTIME filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
FILE_CTIME="${FILE_CTIME%%.*}"
|
FILE_CTIME="${FILE_CTIME%%.*}"
|
||||||
@@ -474,12 +498,14 @@ while read -r FILE_SIZE FILE_CTIME filepath; do
|
|||||||
warn "$ICON_TRASH ORPHAN: $filepath"
|
warn "$ICON_TRASH ORPHAN: $filepath"
|
||||||
(( ORPHAN_COUNT++ ))
|
(( ORPHAN_COUNT++ ))
|
||||||
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
ORPHAN_BYTES=$(( ORPHAN_BYTES + FILE_SIZE ))
|
||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||||
|
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||||
else
|
else
|
||||||
log "JUNK: $filepath"
|
log "JUNK: $filepath"
|
||||||
(( JUNK_COUNT++ ))
|
(( JUNK_COUNT++ ))
|
||||||
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
JUNK_BYTES=$(( JUNK_BYTES + FILE_SIZE ))
|
||||||
echo "$filepath" >> "$TO_DELETE_FILE"
|
if ! orphan_strike_ok "$filepath"; then (( HELD_COUNT++ )); HELD_BYTES=$(( HELD_BYTES + FILE_SIZE )); continue; fi
|
||||||
|
printf '%s\t%s\t%s\n' "$FILE_SIZE" "$FILE_CTIME" "$filepath" >> "$TO_DELETE_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
# -printf gets size + mtime directly from find's own stat() during the walk, instead of a
|
||||||
@@ -491,13 +517,43 @@ done < <(
|
|||||||
done | sort -u
|
done | sort -u
|
||||||
)
|
)
|
||||||
|
|
||||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
# Eligible, not classified: a file still serving its strikes is an orphan but is not queued this
|
||||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
# run, so it must not appear in the denominator the budget reports against.
|
||||||
|
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES - HELD_BYTES ))
|
||||||
|
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT - HELD_COUNT ))
|
||||||
|
|
||||||
|
# Rebuilt, never edited. Skipped on a dry run: a preview that advanced real counters would make
|
||||||
|
# the next real run delete a run early.
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
mv "$STRIKES_NEW" "$STRIKES_FILE" 2>/dev/null || warn "Could not update $STRIKES_FILE"
|
||||||
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
# ━━━ Safety Layer 7 — Deletion Size Threshold ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
check_delete_size_threshold "$TOTAL_DELETE_BYTES" "$SONARR_MAX_DELETE_GB" "Sonarr Cleanup"
|
# A per-run budget, not a veto — see apply_delete_budget() in common.sh. The ceiling still caps
|
||||||
|
# any single run; it just no longer deadlocks on a backlog larger than itself.
|
||||||
|
BUDGET_FILE="$TMP_DIR/to_delete_budgeted.txt"
|
||||||
|
|
||||||
|
if [[ "$I_KNOW" == true ]]; then
|
||||||
|
warn "OVERRIDE — --i-know-what-im-doing active, per-run budget not applied"
|
||||||
|
cut -d"$(printf '\t')" -f3- "$TO_DELETE_FILE" > "$BUDGET_FILE"
|
||||||
|
_BUDGET_KEPT_COUNT=$TOTAL_REMOVED; _BUDGET_KEPT_BYTES=$TOTAL_DELETE_BYTES
|
||||||
|
_BUDGET_DEFERRED_COUNT=0; _BUDGET_DEFERRED_BYTES=0; _BUDGET_STUCK=""
|
||||||
|
else
|
||||||
|
apply_delete_budget "$TO_DELETE_FILE" "$BUDGET_FILE" "$SONARR_MAX_DELETE_GB"
|
||||||
|
if [[ -n "$_BUDGET_STUCK" ]]; then
|
||||||
|
error "Single file exceeds the ${SONARR_MAX_DELETE_GB}GB budget on its own — nothing removed this run"
|
||||||
|
error " $_BUDGET_STUCK"
|
||||||
|
error "Raise SONARR_MAX_DELETE_GB or clear this one with --i-know-what-im-doing"
|
||||||
|
notify "Sonarr cleanup stalled on $(hostname) — one file exceeds the ${SONARR_MAX_DELETE_GB}GB budget" \
|
||||||
|
"Sonarr Cleanup" "warning"
|
||||||
|
elif [[ "$_BUDGET_DEFERRED_COUNT" -gt 0 ]]; then
|
||||||
|
warn "Budget ${SONARR_MAX_DELETE_GB}GB — removing $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED ($(format_bytes "$_BUDGET_KEPT_BYTES")), deferring $_BUDGET_DEFERRED_COUNT ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) to the next run"
|
||||||
|
notify "Sonarr cleanup removed $(format_bytes "$_BUDGET_KEPT_BYTES") of $(format_bytes "$TOTAL_DELETE_BYTES") on $(hostname) — $_BUDGET_DEFERRED_COUNT file(s) deferred" \
|
||||||
|
"Sonarr Cleanup" "normal"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
# ── Execute Deletions ─────────────────────────────────────────────────────────────────────────
|
||||||
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
# Reuses TO_DELETE_FILE from the classification pass above instead of re-walking and
|
||||||
@@ -506,7 +562,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
|||||||
while IFS= read -r filepath; do
|
while IFS= read -r filepath; do
|
||||||
[[ -z "$filepath" ]] && continue
|
[[ -z "$filepath" ]] && continue
|
||||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||||
done < "$TO_DELETE_FILE"
|
done < "$BUDGET_FILE"
|
||||||
|
|
||||||
info "Cleaning up empty folders..."
|
info "Cleaning up empty folders..."
|
||||||
for host_path in "${SCAN_ROOTS[@]}"; do
|
for host_path in "${SCAN_ROOTS[@]}"; do
|
||||||
@@ -532,6 +588,10 @@ echo "$ICON_SHIELD Protected: $PROTECTED_COUNT files (artwork, subtitles,
|
|||||||
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
echo "$ICON_TRASH Orphans: $ORPHAN_COUNT files ($ORPHAN_HUMAN)"
|
||||||
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
echo "$ICON_TRASH Junk: $JUNK_COUNT files ($JUNK_HUMAN)"
|
||||||
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
echo "$ICON_SKIP Recent skipped: $RECENT_COUNT files (under ${SONARR_ORPHAN_AGE} days)"
|
||||||
|
[[ "${HELD_COUNT:-0}" -gt 0 ]] && \
|
||||||
|
echo "$ICON_SKIP Held (strikes): $HELD_COUNT files ($(format_bytes "$HELD_BYTES")) — under ${SONARR_ORPHAN_STRIKE_LIMIT} consecutive runs"
|
||||||
|
[[ "${_BUDGET_DEFERRED_COUNT:-0}" -gt 0 ]] && \
|
||||||
|
echo "$ICON_SKIP Deferred: $_BUDGET_DEFERRED_COUNT files ($(format_bytes "$_BUDGET_DEFERRED_BYTES")) — over the ${SONARR_MAX_DELETE_GB}GB run budget"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
@@ -540,8 +600,10 @@ if [[ "$DRY_RUN" == true ]]; then
|
|||||||
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
elif [[ "$TOTAL_REMOVED" -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Clean — nothing to remove"
|
echo "$ICON_DONE Clean — nothing to remove"
|
||||||
else
|
else
|
||||||
warn "$ICON_DONE Removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
# What was actually removed, not what was classified. With strikes and a budget in force those
|
||||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
# differ, and reporting the classification as the outcome is the oldest bug shape here.
|
||||||
|
warn "$ICON_DONE Removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files ($(format_bytes "$_BUDGET_KEPT_BYTES"))"
|
||||||
|
notify "Sonarr cleanup on $(hostname) — removed $_BUDGET_KEPT_COUNT of $TOTAL_REMOVED eligible files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" \
|
||||||
"Sonarr Cleanup" "warning"
|
"Sonarr Cleanup" "warning"
|
||||||
# Notify Emby to clean missing files — removes ghost entries immediately
|
# Notify Emby to clean missing files — removes ghost entries immediately
|
||||||
notify_emby_scan
|
notify_emby_scan
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
# EMPTY fields, never overwrites existing values unless --overwrite is passed.
|
# EMPTY fields, never overwrites existing values unless --overwrite is passed.
|
||||||
#
|
#
|
||||||
# After populating, pushes the updated conf to all partners via conf_sync.sh
|
# After populating, pushes the updated conf to all partners via conf_sync.sh
|
||||||
# so they have the fresh keys in their /tmp/.cache/vv/d/ cache immediately.
|
# so they have the fresh keys in their /tmp/varaverk/conf/ cache immediately.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# AUTO-DETECTED FIELDS
|
# AUTO-DETECTED FIELDS
|
||||||
@@ -158,7 +158,11 @@
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
source "$SCRIPT_DIR/../../../load_config.sh"
|
# Deployment/ sits one level under the repo root, not three. The old ../../../ resolved to
|
||||||
|
# /boot/config on a flash install and /mnt/user on an appdata one — outside the repo either
|
||||||
|
# way, so this sourced nothing and every helper below was "command not found". Onboard Step 11
|
||||||
|
# has been failing on that since it was written.
|
||||||
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
SCRIPTS_ROOT="$SCRIPTS_DIR"
|
SCRIPTS_ROOT="$SCRIPTS_DIR"
|
||||||
|
|
||||||
OVERWRITE=false
|
OVERWRITE=false
|
||||||
|
|||||||
@@ -95,7 +95,7 @@
|
|||||||
|
|
||||||
# ━━━ Identity ━━━
|
# ━━━ Identity ━━━
|
||||||
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
|
# HOSTN hostname lives in master.conf (not a credential — safe for all servers).
|
||||||
# SSH key used for all server-to-server operations — rsync, failover, conf sync.
|
# SSH key used for all server-to-server operations — rsync, fallback, conf sync.
|
||||||
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
|
# Convention: /root/.ssh/<hostname-lowercase-no-unraid-prefix>_rsync_automation
|
||||||
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
|
# Must be in /root/.ssh/ and authorised in the partner's /root/.ssh/authorized_keys.
|
||||||
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
|
# Run Partnership/ssh_setup.sh to generate the key and copy it to the partner.
|
||||||
@@ -113,6 +113,11 @@
|
|||||||
HOSTN_EMBY_CONTAINER="Emby"
|
HOSTN_EMBY_CONTAINER="Emby"
|
||||||
HOSTN_EMBY_URL="http://localhost:8096"
|
HOSTN_EMBY_URL="http://localhost:8096"
|
||||||
HOSTN_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
HOSTN_EMBY_API_KEY="" # Emby Dashboard → API Keys → + New Key
|
||||||
|
HOSTN_EMBY_PUBLIC_URL="" # e.g. https://media.example.com/emby — browser-reachable base, used
|
||||||
|
# to build image URLs that render in the WebGUI. Deliberately separate
|
||||||
|
# from HOSTN_EMBY_URL: that one is for server-side API calls and is
|
||||||
|
# usually localhost, which resolves to the wrong machine in a browser.
|
||||||
|
# Empty = features that need an image quietly go without one.
|
||||||
|
|
||||||
# ━━━ Jellyfin ━━━
|
# ━━━ Jellyfin ━━━
|
||||||
HOSTN_JELLYFIN_CONTAINER="Jellyfin"
|
HOSTN_JELLYFIN_CONTAINER="Jellyfin"
|
||||||
@@ -150,6 +155,23 @@
|
|||||||
# "Authelia|9091"
|
# "Authelia|9091"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Shares rsynced to the mirror during onboard Step 1e, BEFORE the auth containers are created.
|
||||||
|
# This is the only rsync an onboard performs — media is never seeded here.
|
||||||
|
# Profile is inferred from the directory basename, so Critical-Data resolves to critical-data:
|
||||||
|
# a clean copy with the auth containers stopped on both sides. Do not point this at a share
|
||||||
|
# whose profile keeps databases running; a dirty copy of MariaDB or Redis is worse than none,
|
||||||
|
# because the container starts, reads Up, and restarts a dead database behind it.
|
||||||
|
HOSTN_PARTNERSHIP_PROVISION_SHARES=(
|
||||||
|
"/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — the auth stack
|
||||||
|
)
|
||||||
|
|
||||||
|
# Containers that belong in "<PartnerShort>-Fallback" on the mirror rather than in a mirrored
|
||||||
|
# copy of this host's folder layout — the ones that exist there only to cover this host going
|
||||||
|
# dark. Everything else the onboard deploys is filed onto the same shelf it occupies here
|
||||||
|
# (Arrs Stack, Networking, Databases…), because it runs on the mirror continuously.
|
||||||
|
# Empty is the normal state: leave it empty unless a container is genuinely failover-only.
|
||||||
|
HOSTN_PARTNERSHIP_FALLBACK_ONLY=()
|
||||||
|
|
||||||
# XML templates pushed to mirror during onboard — auth stack.
|
# XML templates pushed to mirror during onboard — auth stack.
|
||||||
# Dependencies (databases) must come before apps that depend on them.
|
# Dependencies (databases) must come before apps that depend on them.
|
||||||
HOSTN_PARTNERSHIP_AUTH_STACK=(
|
HOSTN_PARTNERSHIP_AUTH_STACK=(
|
||||||
@@ -465,6 +487,8 @@
|
|||||||
HOSTN_RADARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated anime root
|
HOSTN_RADARR_ANIME_ROOT="" # rootFolderPath literal, as reported by Radarr API — leave blank if no dedicated anime root
|
||||||
HOSTN_RADARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Radarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Movies") — blank disables the download orphan cleaner for Radarr
|
HOSTN_RADARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Radarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Movies") — blank disables the download orphan cleaner for Radarr
|
||||||
HOSTN_RADARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Radarr's container sees it (e.g. "/downloads/Completed/Movies") — needed to trigger import scans on held folders
|
HOSTN_RADARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Radarr's container sees it (e.g. "/downloads/Completed/Movies") — needed to trigger import scans on held folders
|
||||||
|
HOSTN_LIDARR_DOWNLOAD_DIR="" # host path of the completed-downloads folder Lidarr imports from (e.g. "/mnt/cache/Temp_Storage/SABnzbd/Completed/Music") — blank disables the download orphan cleaner for Lidarr
|
||||||
|
HOSTN_LIDARR_DOWNLOAD_CONTAINER_DIR="" # same folder as Lidarr's container sees it (e.g. "/downloads/Completed/Music") — needed to trigger import scans on held folders
|
||||||
|
|
||||||
declare -A HOSTN_RADARR_PATH_MAP=(
|
declare -A HOSTN_RADARR_PATH_MAP=(
|
||||||
# ["/movies"]="/mnt/user/Movies"
|
# ["/movies"]="/mnt/user/Movies"
|
||||||
|
|||||||
+157
-27
@@ -197,13 +197,20 @@
|
|||||||
# ── Conf Sync ──
|
# ── Conf Sync ──
|
||||||
# conf_sync.sh pulls partner host confs into /tmp/.vv/ RAM cache on array start and after
|
# conf_sync.sh pulls partner host confs into /tmp/.vv/ RAM cache on array start and after
|
||||||
# any conf save — makes partner vars (HOST2_*, etc.) available without touching git.
|
# any conf save — makes partner vars (HOST2_*, etc.) available without touching git.
|
||||||
CONF_SYNC_ENABLED=true
|
# Ships OFF for the same reason as RSYNC_ENABLED: a node seeded from this template has not yet
|
||||||
|
# been paired with anyone, and pulling a partner's confs before the partnership exists is either
|
||||||
|
# a no-op or a surprise. The onboard flow arms it once the pairing is real.
|
||||||
|
CONF_SYNC_ENABLED=false
|
||||||
|
|
||||||
# ── Arr Sync ──
|
# ── Arr Sync ──
|
||||||
# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
# arr_sync.sh syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
||||||
# Runs before rsync — all nodes agree on tracked library before files are transferred.
|
# Runs before rsync — all nodes agree on tracked library before files are transferred.
|
||||||
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
|
# Remote API keys are read live from each node's config.xml via SSH — never stored here.
|
||||||
ARR_SYNC_ENABLED=true
|
# Ships OFF. Bidirectional means a fresh node with an empty library is an equal party to the
|
||||||
|
# merge, and "all nodes agree on tracked library" is not something you want decided by the node
|
||||||
|
# that knows nothing. Onboard Step 9 runs arr_sync explicitly during bootstrap; this gate is for
|
||||||
|
# the unattended runs afterwards.
|
||||||
|
ARR_SYNC_ENABLED=false
|
||||||
ARR_SYNC_BLOCKLIST="${DB_DIR}/arr_sync_blocklist.tsv"
|
ARR_SYNC_BLOCKLIST="${DB_DIR}/arr_sync_blocklist.tsv"
|
||||||
ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node
|
ARR_SYNC_CONNECT_TIMEOUT=10 # seconds — SSH connect timeout per node
|
||||||
ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches
|
ARR_SYNC_API_TIMEOUT=60 # seconds — curl timeout for library fetches
|
||||||
@@ -266,8 +273,10 @@
|
|||||||
VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"
|
VARAVERK_SETUP_FILE="$STATE_DIR/varaverk_setup.db"
|
||||||
|
|
||||||
# Tailscale removal on offboard.
|
# Tailscale removal on offboard.
|
||||||
PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard
|
PARTNERSHIP_REMOVE_TAILSCALE=false # opt-in — removes the mirror from the tailnet on offboard.
|
||||||
# false = skip removal (manual or testing)
|
# OFF by default: it needs a TAILSCALE_API_KEY, which
|
||||||
|
# expires and must be reissued, and a feature that
|
||||||
|
# silently no-ops on a stale key should not be opt-out.
|
||||||
|
|
||||||
# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
|
# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true.
|
||||||
# Stays in shared conf — only owner uses it, and owner is always running this script.
|
# Stays in shared conf — only owner uses it, and owner is always running this script.
|
||||||
@@ -338,7 +347,16 @@
|
|||||||
GITEA_CONTAINER="Gitea"
|
GITEA_CONTAINER="Gitea"
|
||||||
GITEA_REPO_PATH="" # e.g. YourUser/Varaverk.git
|
GITEA_REPO_PATH="" # e.g. YourUser/Varaverk.git
|
||||||
GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS
|
GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS
|
||||||
TARGET_DIR="/mnt/user/appdata/Varaverk"
|
# Derived, never a literal. This is the directory git_pull_execute.sh pulls into, and it is the
|
||||||
|
# one host-specific path that used to live in master.conf as a fixed string — which made the
|
||||||
|
# shared conf unshareable: pushing it to a partner that had migrated its install repointed that
|
||||||
|
# partner's pull target at the old location, and the next pull built a second copy there while
|
||||||
|
# the live install ran somewhere else.
|
||||||
|
#
|
||||||
|
# SCRIPTS_DIR is exported by load_config.sh from its own location before this file is sourced, so
|
||||||
|
# this resolves to wherever the install actually is, on every node, in either storage mode, with
|
||||||
|
# nothing to keep in step. DATA_DIR and STATE_DIR below already worked this way.
|
||||||
|
TARGET_DIR="${SCRIPTS_DIR}"
|
||||||
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
GITEA_SSH_KEY="/root/.ssh/unraid_gitea"
|
||||||
SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222)
|
SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222)
|
||||||
GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh
|
GITEA_HTTP_PORT=3000 # Gitea web/API port — used by gitea_ssh_setup.sh
|
||||||
@@ -354,7 +372,7 @@
|
|||||||
# Scripts launched by array_started.sh when the array comes online.
|
# Scripts launched by array_started.sh when the array comes online.
|
||||||
# Launched in order — each as a background process.
|
# Launched in order — each as a background process.
|
||||||
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
|
||||||
# Continuous scripts (failover) run until array stops.
|
# Continuous scripts (fallback) run until array stops.
|
||||||
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
|
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
|
||||||
# watchdog_orchestrator.sh — NOT launched here.
|
# watchdog_orchestrator.sh — NOT launched here.
|
||||||
ARRAY_START_SCRIPTS=(
|
ARRAY_START_SCRIPTS=(
|
||||||
@@ -453,7 +471,6 @@
|
|||||||
# to 1 day). 30min was true "always fresh" but had no consumer that needed it that often.
|
# to 1 day). 30min was true "always fresh" but had no consumer that needed it that often.
|
||||||
"Arrs_Stack/arr_cache_prefill.sh ARR_PREFILL_WAIT_MINUTES=1" # keep the shared arr tracked-data cache fresh
|
"Arrs_Stack/arr_cache_prefill.sh ARR_PREFILL_WAIT_MINUTES=1" # keep the shared arr tracked-data cache fresh
|
||||||
"Arrs_Stack/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items
|
"Arrs_Stack/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items
|
||||||
"AI/ai_token_sync.sh" # pull partner AI token ledgers into the tmpfs cache
|
|
||||||
)
|
)
|
||||||
# arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
|
# arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
|
||||||
# It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
|
# It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
|
||||||
@@ -473,9 +490,9 @@
|
|||||||
"Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files
|
"Arrs_Stack/sonarr_cleanup.sh" # remove orphaned TV files
|
||||||
"Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files
|
"Arrs_Stack/radarr_cleanup.sh" # remove orphaned movie files
|
||||||
# Daily, not weekly — DOWNLOAD_ORPHAN_AGE gates each folder on its own mtime, so
|
# Daily, not weekly — DOWNLOAD_ORPHAN_AGE gates each folder on its own mtime, so
|
||||||
# frequency costs nothing in safety, but a weekly run clears a whole week of
|
# frequency costs nothing in safety, while a weekly run makes each pass a whole
|
||||||
# newly-eligible orphans at once and can exceed DOWNLOAD_ORPHAN_MAX_DELETE_GB, which
|
# week of newly-eligible orphans at once and far more likely to spill past
|
||||||
# aborts the pass entirely and rolls the backlog into an even larger next run.
|
# DOWNLOAD_ORPHAN_MAX_DELETE_GB into a deferred remainder.
|
||||||
"Arrs_Stack/arr_download_orphan_cleaner.sh" # sweep orphaned completed downloads out of the SAB Completed folders — deletes junk + already-imported leftovers, triggers import scans for genuinely-missing content
|
"Arrs_Stack/arr_download_orphan_cleaner.sh" # sweep orphaned completed downloads out of the SAB Completed folders — deletes junk + already-imported leftovers, triggers import scans for genuinely-missing content
|
||||||
# Runs after cleanup, not before — frees disk space from orphans/junk first, so the
|
# Runs after cleanup, not before — frees disk space from orphans/junk first, so the
|
||||||
# searches triggered below (for relocated hasFile=false/episodeFileCount=0 entries)
|
# searches triggered below (for relocated hasFile=false/episodeFileCount=0 entries)
|
||||||
@@ -486,6 +503,11 @@
|
|||||||
"Arrs_Stack/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
"Arrs_Stack/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||||
"Arrs_Stack/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
"Arrs_Stack/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||||
"Arrs_Stack/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
"Arrs_Stack/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||||
|
# Before the two docker jobs below, deliberately: they pull images and restart
|
||||||
|
# containers, so a container can legitimately not exist for a moment while they run.
|
||||||
|
# Reading docker in its settled overnight state keeps a rebuild from costing a strike.
|
||||||
|
# Three strikes at this cadence means three days genuinely absent before conf is edited.
|
||||||
|
"Tools/conf_container_prune.sh" # drop containers that no longer exist from the conf lists
|
||||||
"Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
"Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
||||||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||||||
# Last, and after git_pull_execute.sh which must always run first. Daily rather than with
|
# Last, and after git_pull_execute.sh which must always run first. Daily rather than with
|
||||||
@@ -614,7 +636,7 @@
|
|||||||
# Fine grained control — disable specific orchestrators while keeping others
|
# Fine grained control — disable specific orchestrators while keeping others
|
||||||
# Use when: rebuilding secondary, testing, per-window bandwidth management
|
# Use when: rebuilding secondary, testing, per-window bandwidth management
|
||||||
#
|
#
|
||||||
# Example — HOST2 data rebuild:
|
# Example — rebuilding a partner's data:
|
||||||
# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
||||||
# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs
|
# CRITICAL_RSYNC_ENABLED=true ← 30min auth stack sync still runs
|
||||||
# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild
|
# INTERMEDIATE_RSYNC_ENABLED=false ← skip 4h arr/mid-day rsync during rebuild
|
||||||
@@ -622,15 +644,37 @@
|
|||||||
# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe)
|
# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe)
|
||||||
# MONTHLY_RSYNC_ENABLED=true ← monthly_maintenance.sh rsync section
|
# MONTHLY_RSYNC_ENABLED=true ← monthly_maintenance.sh rsync section
|
||||||
# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works
|
# FALLBACK_RSYNC_ENABLED=true ← handback writeback still works
|
||||||
|
# MEDIA_SEED_ENABLED=false ← the onboard first-fill never starts
|
||||||
|
#
|
||||||
|
# MEDIA_SEED_ENABLED is the odd one out: it does not gate a schedule, it gates a single
|
||||||
|
# multi-week transfer. Rsync/media_seed.sh pushes every DAILY_SYNC_SHARES entry to a newly
|
||||||
|
# onboarded partner — on HOST1 that is ~28 TB against BW_LIMIT, so weeks. Turn it off when the
|
||||||
|
# partner is going to be filled some other way (a physically moved disk, an existing library),
|
||||||
|
# and onboard will finish without ever starting it. Turning it back on does not start anything
|
||||||
|
# by itself; dispatch it from the Partnership tab or run the script.
|
||||||
|
#
|
||||||
|
# NOTE ON THE DEFAULT: this one ships OFF, and unlike Tier 1 it is not armed by a successful
|
||||||
|
# onboard — Step 9c arms RSYNC_ENABLED, CONF_SYNC_ENABLED and ARR_SYNC_ENABLED, and
|
||||||
|
# deliberately not this. Seeding a partner is a decision about weeks of bandwidth and tens of
|
||||||
|
# terabytes of disk, and the node that has just been onboarded is exactly the node nobody has
|
||||||
|
# checked the free space on yet. Arm it when you have decided the partner should be filled by
|
||||||
|
# rsync rather than by hand. Turning it on is one switch; noticing it was on is not.
|
||||||
# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
|
# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
|
||||||
# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true
|
# → When ready: INTERMEDIATE_RSYNC_ENABLED=true DAILY_RSYNC_ENABLED=true
|
||||||
RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
# NOTE ON THE DEFAULT: Tier 1 ships OFF. This template is what a brand-new node is seeded from
|
||||||
|
# (Manual-Deployment.md — "Rebuilding a Wiped Node"), and a node that has just been stood up has
|
||||||
|
# empty shares and an unverified partner. Defaulting the global gate ON meant a fresh install
|
||||||
|
# started transferring — with --delete in the mix — before anyone had looked at it. Tier 2 stays
|
||||||
|
# ON so the structure is intact the moment you do arm Tier 1; this mirrors HOST1's live posture.
|
||||||
|
# Changing this default cannot affect an existing node: conf_upgrade KEEPS values already set.
|
||||||
|
RSYNC_ENABLED=false # Tier 1 — global gate, overrides everything below
|
||||||
CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh rsync section
|
CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh rsync section
|
||||||
INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section
|
INTERMEDIATE_RSYNC_ENABLED=true # Tier 2 — intermediate_sync_maintenance.sh rsync section
|
||||||
DAILY_RSYNC_ENABLED=true # Tier 2 — daily_sync_maintenance.sh rsync section
|
DAILY_RSYNC_ENABLED=true # Tier 2 — daily_sync_maintenance.sh rsync section
|
||||||
WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
||||||
MONTHLY_RSYNC_ENABLED=true # Tier 2 — monthly_maintenance.sh rsync section
|
MONTHLY_RSYNC_ENABLED=true # Tier 2 — monthly_maintenance.sh rsync section
|
||||||
FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback
|
FALLBACK_RSYNC_ENABLED=true # Tier 2 — fallback.sh writeback jobs on handback
|
||||||
|
MEDIA_SEED_ENABLED=false # Tier 2 — Rsync/media_seed.sh, the onboard first-fill
|
||||||
|
|
||||||
# ━━━ Download Webhook ━━━
|
# ━━━ Download Webhook ━━━
|
||||||
# Immediate push to remote nodes on every Sonarr/Radarr/Lidarr Download event.
|
# Immediate push to remote nodes on every Sonarr/Radarr/Lidarr Download event.
|
||||||
@@ -654,6 +698,13 @@
|
|||||||
# lowered from 23 on 2026-07-16 — daily maintenance jobs alone now
|
# lowered from 23 on 2026-07-16 — daily maintenance jobs alone now
|
||||||
# take ~4h; 19h cap leaves room for them to still run same-day
|
# take ~4h; 19h cap leaves room for them to still run same-day
|
||||||
# before the next 1am fire even if a share hits the cap
|
# before the next 1am fire even if a share hits the cap
|
||||||
|
RSYNC_MIN_FREE_GB=50 # warn if any disk backing the remote share is under this
|
||||||
|
# (0 disables). Online is not writable: a full destination
|
||||||
|
# makes rsync stall rather than fail. Warns, never aborts.
|
||||||
|
CONF_PRUNE_STRIKE_LIMIT=3 # consecutive runs a container must be absent from docker
|
||||||
|
# before its name is removed from the conf lists. A
|
||||||
|
# rebuild makes a container vanish briefly; one miss
|
||||||
|
# must never be enough to edit conf.
|
||||||
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override
|
||||||
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override
|
||||||
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
CONTAINER_DELAY=5 # seconds before starting delayed containers
|
||||||
@@ -756,7 +807,25 @@
|
|||||||
declare -A PROFILE_EXCLUDE_DIRS=(
|
declare -A PROFILE_EXCLUDE_DIRS=(
|
||||||
[arrs_stack]="logs *.tmp"
|
[arrs_stack]="logs *.tmp"
|
||||||
[critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
[critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
||||||
[critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm"
|
# appendonlydir and mariadb-bin.* are excluded for the same reason as *.db-wal: they are
|
||||||
|
# written continuously by a database this profile copies WHILE IT RUNS. The exclusion had
|
||||||
|
# SQLite's WAL and nothing for Redis or MariaDB, and both broke the mirror on 2026-08-17.
|
||||||
|
#
|
||||||
|
# The failure is not a torn file, it is a surviving pointer to a missing one. The manifest
|
||||||
|
# and the index are a few hundred bytes and land instantly; the data files they name are
|
||||||
|
# large and are still in flight. The mirror ends up holding a promise nothing can keep:
|
||||||
|
# Redis appendonly.aof.manifest → appendonly.aof.30.base.rdb (absent) → exit 1
|
||||||
|
# MariaDB mariadb-bin.index, 265 → mariadb-bin.000265 (absent) → Aborting
|
||||||
|
#
|
||||||
|
# Excluded rather than repaired: neither is needed for the engine to START. Redis rebuilds
|
||||||
|
# an empty AOF, MariaDB begins a new binlog sequence. On a fallback mirror that is the
|
||||||
|
# right trade — those files exist for persistence and replication, and a database that
|
||||||
|
# starts empty beats one that cannot start at all.
|
||||||
|
#
|
||||||
|
# NOT a guarantee that a live-copied InnoDB is consistent. It happened to survive both
|
||||||
|
# times. The honest fix is a DB-layer dump synced as a file; this stops the observed
|
||||||
|
# breakage without pretending to be that.
|
||||||
|
[critical-fallback]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt *.db-wal *.db-shm appendonlydir mariadb-bin.*"
|
||||||
[important-data]="logs *.tmp"
|
[important-data]="logs *.tmp"
|
||||||
[emby]="logs transcodes cache crash*"
|
[emby]="logs transcodes cache crash*"
|
||||||
)
|
)
|
||||||
@@ -791,7 +860,34 @@
|
|||||||
|
|
||||||
EXTERNAL_IP="8.8.8.8"
|
EXTERNAL_IP="8.8.8.8"
|
||||||
FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
|
FALLBACK_CHECK_INTERVAL=30 # seconds between fallback state checks
|
||||||
FALLBACK_HANDBACK_STRIKES=3 # consecutive healthy checks before initiating handback (3×30s = 90s)
|
# Cost measured 2026-08-22: ~15ms CPU and 784 bytes per cycle
|
||||||
|
# per host (~2 MB/day, half of it across the mesh). The 2s of
|
||||||
|
# wall time per cycle is ping waiting, not work.
|
||||||
|
FALLBACK_HANDBACK_STRIKES=4 # consecutive healthy checks before initiating handback
|
||||||
|
# 4×30s = 2min. Raising this costs nothing — it requires more
|
||||||
|
# of the checks that already happen, it does not add any.
|
||||||
|
# Any failed check resets the counter, so the window is a
|
||||||
|
# genuine "stayed up this long", not "was up this often".
|
||||||
|
# Entry is deliberately unguarded — one false from ping_remote enters FALLBACK, with no strike
|
||||||
|
# count, because every second of a real outage is downtime and DDNS TTL is racing it. The
|
||||||
|
# confidence comes from the probe itself instead: see FALLBACK_PROBE_* below and _ping_retry()
|
||||||
|
# in common.sh.
|
||||||
|
FALLBACK_PROBE_RETRIES=3 # reachability attempts before declaring a host down
|
||||||
|
FALLBACK_PROBE_RETRY_DELAY=2 # seconds between those attempts
|
||||||
|
# Success returns on the first try, so a healthy cycle costs
|
||||||
|
# nothing extra. Only a failure pays: 3×4s + 2×2s = 16s worst
|
||||||
|
# case, which must stay under FALLBACK_CHECK_INTERVAL.
|
||||||
|
FALLBACK_DRY_RUN=false # run the daemon in PREVIEW mode, permanently
|
||||||
|
# Survives array start, unlike a --dry-run typed at a shell:
|
||||||
|
# array_started.sh launches with no arguments, so a hand-run
|
||||||
|
# preview is replaced by the LIVE daemon at the next boot.
|
||||||
|
# Decides nothing, starts nothing, moves no DNS, sends no
|
||||||
|
# notification — it only reports what it WOULD have done.
|
||||||
|
# Set true to observe real outages before arming for real.
|
||||||
|
FALLBACK_LOG_MAX_MB=5 # size cap for data/logs/fallback.log before one rotation
|
||||||
|
# Event-only output (the default, no --log) is a few lines
|
||||||
|
# per incident and holds years; --log fills this in about a
|
||||||
|
# fortnight and keeps the previous one as fallback.log.1
|
||||||
FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
|
FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db"
|
||||||
FALLBACK_ENABLED=false # set true once both servers are configured and paired
|
FALLBACK_ENABLED=false # set true once both servers are configured and paired
|
||||||
# false = suppresses "not running" warnings in status scripts
|
# false = suppresses "not running" warnings in status scripts
|
||||||
@@ -1172,6 +1268,8 @@
|
|||||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
|
LIDARR_TRACKED_COUNT_FILE="${DB_DIR}/lidarr_tracked.count"
|
||||||
|
LIDARR_ORPHAN_STRIKES_FILE="${DB_DIR}/lidarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||||
|
LIDARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||||
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
LIDARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
|
|
||||||
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
# Lidarr tracked-data cache — shared by lidarr_cleanup.sh, lidarr_duplicate_artist_cleanup.sh,
|
||||||
@@ -1181,7 +1279,7 @@
|
|||||||
LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db"
|
LIDARR_RESCAN_DURATION_DB="${DB_DIR}/lidarr_rescan_duration.db"
|
||||||
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
|
LIDARR_CACHE_MAX_AGE_DAYS=1 # force a live refresh (or rescan-aware wait) past this age
|
||||||
ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr
|
ARR_PREFILL_WAIT_MINUTES=10 # array-start prefill: how long to retry reaching each arr
|
||||||
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma")
|
LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma" "wv" "ape" "aiff" "aif" "alac" "dsf" "dff" "mpc" "tta")
|
||||||
LIDARR_PROTECTED_PATTERNS=(
|
LIDARR_PROTECTED_PATTERNS=(
|
||||||
# Metadata
|
# Metadata
|
||||||
"*.nfo" "*.tbn"
|
"*.nfo" "*.tbn"
|
||||||
@@ -1247,6 +1345,8 @@
|
|||||||
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
SONARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
|
SONARR_TRACKED_COUNT_FILE="${DB_DIR}/sonarr_tracked.count"
|
||||||
|
SONARR_ORPHAN_STRIKES_FILE="${DB_DIR}/sonarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||||
|
SONARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify before it is removed
|
||||||
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
SONARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
|
SONARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveSeries command to
|
||||||
# reach "completed" — generous because a large series can sit
|
# reach "completed" — generous because a large series can sit
|
||||||
@@ -1259,6 +1359,15 @@
|
|||||||
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
|
# against a one-off ffprobe hiccup (mid-write file, NFS blip)
|
||||||
# triggering an unnecessary delete. Resets to 0 the moment a
|
# triggering an unnecessary delete. Resets to 0 the moment a
|
||||||
# file probes clean again.
|
# file probes clean again.
|
||||||
|
CORRUPTION_SCAN_MAX_CORRUPT_PCT=10 # abort the run, committing nothing, if this share of
|
||||||
|
# newly-scanned files reads as corrupt. A healthy library sits
|
||||||
|
# near zero; a high rate means the detector is wrong, not the
|
||||||
|
# library. Only counts ffprobe exit != 0.
|
||||||
|
CORRUPTION_SCAN_MAX_CONSECUTIVE=15 # abort after this many files in a row fail to probe
|
||||||
|
# cleanly — catches the ffprobe container dying mid-scan,
|
||||||
|
# which the pre-flight health check cannot see.
|
||||||
|
CORRUPTION_SCAN_GUARD_MIN_SCANNED=20 # below this many newly-scanned files the percentage
|
||||||
|
# ceiling is not applied — too small a sample to judge.
|
||||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||||
SONARR_PROTECTED_PATTERNS=(
|
SONARR_PROTECTED_PATTERNS=(
|
||||||
# Subtitles
|
# Subtitles
|
||||||
@@ -1285,6 +1394,12 @@
|
|||||||
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
RADARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||||
# protects against API returning partial data on a bad day
|
# protects against API returning partial data on a bad day
|
||||||
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
|
RADARR_TRACKED_COUNT_FILE="${DB_DIR}/radarr_tracked.count"
|
||||||
|
RADARR_ORPHAN_STRIKES_FILE="${DB_DIR}/radarr_orphan_strikes.tsv" # consecutive-classification counts, keyed by host path
|
||||||
|
RADARR_ORPHAN_STRIKE_LIMIT=2 # consecutive runs a file must classify for deletion before it is
|
||||||
|
# removed. Gate 6 already catches an API returning far too few
|
||||||
|
# tracked files; this catches the partial failure too small to trip
|
||||||
|
# that percentage — one root folder failing to enumerate makes its
|
||||||
|
# movies look orphaned, and a transient one will not repeat.
|
||||||
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
RADARR_IMPORT_SCAN_TIMEOUT=600 # seconds to wait for pre-flight import scan
|
||||||
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
|
RADARR_MOVE_POLL_TIMEOUT=3600 # seconds to wait for a single async MoveMovie command to
|
||||||
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
|
# reach "completed" — mirrors SONARR_MOVE_POLL_TIMEOUT
|
||||||
@@ -1305,10 +1420,17 @@
|
|||||||
# Per-host dirs: HOST*_SONARR_DOWNLOAD_DIR / HOST*_RADARR_DOWNLOAD_DIR (+ _CONTAINER_DIR).
|
# Per-host dirs: HOST*_SONARR_DOWNLOAD_DIR / HOST*_RADARR_DOWNLOAD_DIR (+ _CONTAINER_DIR).
|
||||||
DOWNLOAD_ORPHAN_CLEANER_ENABLED=true
|
DOWNLOAD_ORPHAN_CLEANER_ENABLED=true
|
||||||
DOWNLOAD_ORPHAN_AGE=7 # days — entries younger than this may be mid-import, never touched
|
DOWNLOAD_ORPHAN_AGE=7 # days — entries younger than this may be mid-import, never touched
|
||||||
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives)
|
DOWNLOAD_ORPHAN_KEEP_MARKER=".vv-keep" # a file by this name inside a download folder pins it —
|
||||||
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # abort delete pass over this — a partial queue fetch would classify
|
# never classified, never deleted. For lossless rips the
|
||||||
# live downloads as orphans, and a huge total is that failure's symptom;
|
# library holds only as MP3, which REDUNDANT would sweep
|
||||||
# --i-know-what-im-doing overrides for known backlogs
|
DOWNLOAD_ORPHAN_MIN_VIDEO_MB=50 # no video file above this = junk (par2 debris, samples, dead archives) — Sonarr/Radarr
|
||||||
|
DOWNLOAD_ORPHAN_MIN_AUDIO_MB=2 # same test for Lidarr — a 50M floor would mark every album folder
|
||||||
|
# as junk, since single tracks rarely reach it
|
||||||
|
DOWNLOAD_ORPHAN_MAX_DELETE_GB=100 # per-run delete budget — a backlog above this drains
|
||||||
|
# safest-first (JUNK, REDUNDANT, then UNMATCHED) up to
|
||||||
|
# the budget and defers the rest to the next run.
|
||||||
|
# Live downloads are protected by DOWNLOAD_ORPHAN_AGE,
|
||||||
|
# not by this; --i-know-what-im-doing clears it in one pass
|
||||||
|
|
||||||
# ━━━ Arr Content Classification (radarr/sonarr_classification_scan.sh) ━━━
|
# ━━━ Arr Content Classification (radarr/sonarr_classification_scan.sh) ━━━
|
||||||
#
|
#
|
||||||
@@ -1501,7 +1623,7 @@
|
|||||||
# request counts. Fifty thousand self-inflicted requests a day would otherwise bury real traffic.
|
# request counts. Fifty thousand self-inflicted requests a day would otherwise bury real traffic.
|
||||||
UPTIME_PROBE_ENABLED=true # master switch for the per-minute probe
|
UPTIME_PROBE_ENABLED=true # master switch for the per-minute probe
|
||||||
UPTIME_PROBE_TIMEOUT=8 # seconds per domain before it counts as down
|
UPTIME_PROBE_TIMEOUT=8 # seconds per domain before it counts as down
|
||||||
UPTIME_PROBE_LIST_TTL=900 # seconds to reuse the domain list from NPM before re-reading it
|
UPTIME_PROBE_LIST_TTL=300 # seconds to reuse the domain list from NPM before re-reading it
|
||||||
|
|
||||||
# ── Auth Sweep ──
|
# ── Auth Sweep ──
|
||||||
# Tools/auth_sweep.sh asks the two questions the Auth tab answers about one host, about every host,
|
# Tools/auth_sweep.sh asks the two questions the Auth tab answers about one host, about every host,
|
||||||
@@ -1544,6 +1666,10 @@
|
|||||||
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
# HOST1_BACKUP_VERIFY_SHARES / HOST2_BACKUP_VERIFY_SHARES
|
||||||
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
BACKUP_VERIFY_SAMPLE=10 # random files to check per share
|
||||||
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample
|
||||||
|
BACKUP_VERIFY_MD5_TIMEOUT_MAX=600 # ceiling for one remote checksum. The per-file budget
|
||||||
|
# scales with size (~50MB/s); this caps it. A fixed
|
||||||
|
# connect-sized timeout killed multi-GB checksums and
|
||||||
|
# the empty result was then reported as MISSING.
|
||||||
|
|
||||||
# ━━━ SMART Health ━━━
|
# ━━━ SMART Health ━━━
|
||||||
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*.
|
||||||
@@ -1908,12 +2034,13 @@
|
|||||||
AI_TOKEN_DB="${AI_DATA_DIR}/ai_token_history.db"
|
AI_TOKEN_DB="${AI_DATA_DIR}/ai_token_history.db"
|
||||||
AI_TOKEN_RETAIN_ROWS=20000 # oldest rows dropped past this — years of ordinary use
|
AI_TOKEN_RETAIN_ROWS=20000 # oldest rows dropped past this — years of ordinary use
|
||||||
|
|
||||||
# AI/ai_token_sync.sh pulls each partner's ledger into the tmpfs cache the tab reads, so the
|
# AI/ai_token_sync.sh pulled each partner's ledger into a tmpfs cache so the fleet total was a
|
||||||
# fleet total is a fleet total. Same trick conf_sync.sh uses for partner confs, minus the push:
|
# fleet total. The mesh shares one AI now: every turn generates on the owner and is recorded
|
||||||
# nothing here is needed by anyone else, so the reader fetches its own data and controls its
|
# there against the node that asked, so the owner's own ledger already holds the whole fleet and
|
||||||
# own freshness. An unreachable partner is a quiet skip, not a warning — a partner is expected
|
# a partner ledger is empty by construction. Left off the schedule rather than deleted — the
|
||||||
# to be down for long stretches, and a four-hourly warn trains you to ignore the script.
|
# reader still merges any cache from before the change, which describes real turns.
|
||||||
AI_TOKEN_SYNC_ENABLED=true
|
AI_TOKEN_SYNC_ENABLED=false
|
||||||
|
|
||||||
|
|
||||||
# ━━━ AI Feature Toggles ━━━
|
# ━━━ AI Feature Toggles ━━━
|
||||||
# Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script
|
# Tier 1 is narration — it cannot change a decision. Tier 2 adds context to a decision a script
|
||||||
@@ -1921,7 +2048,10 @@
|
|||||||
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
|
AI_ASSIST_REPORTS=false # tier 1 — digest / coffee report narration
|
||||||
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
|
AI_ASSIST_WATCHDOG=false # tier 2 — file a finding when a watchdog counter passes its limit (needs AI_REPAIR_ENABLED)
|
||||||
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
|
AI_ASSIST_DISCOVERY=false # tier 2 — discovery / classification judgement calls
|
||||||
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage
|
AI_ASSIST_CLEANUP=false # tier 2 — orphan and stuck-import triage. Describes the shape of a
|
||||||
|
# classification in the log; decides nothing. Off = identical deletions.
|
||||||
|
AI_ASSIST_TIMEOUT=45 # seconds any single assist may take. An assist that can stall a
|
||||||
|
# nightly cleanup is not an assist — it is silently skipped past this.
|
||||||
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
|
AI_ASSIST_ONBOARD=false # tier 3 — onboarding / settings assistance
|
||||||
|
|
||||||
# ━━━ AI Repair ━━━
|
# ━━━ AI Repair ━━━
|
||||||
|
|||||||
@@ -28,6 +28,42 @@
|
|||||||
# pointing at a directory the data is not in is worse than not having started.
|
# pointing at a directory the data is not in is worse than not having started.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Two halves, in order: move the files, then rewrite the conf keys that point at them. Doing it
|
||||||
|
# the other way round would leave every path variable naming a location nothing had reached yet,
|
||||||
|
# and any script that ran in between would create the old layout again underneath the new one.
|
||||||
|
#
|
||||||
|
# Idempotent. A path already under DATA_DIR is left alone, so a re-run after a partial migration
|
||||||
|
# finishes the job rather than moving things twice or failing on what is already done.
|
||||||
|
#
|
||||||
|
# One-time by intent, not by a marker file. There is no "already migrated" flag — the check is
|
||||||
|
# whether each individual path is already where it belongs, which is also what makes an
|
||||||
|
# interrupted run safe to repeat.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Existing keys are rewritten, which is why conf_upgrade cannot do this.
|
||||||
|
# conf_upgrade adds keys the template has and the installation does not, and never rewrites a
|
||||||
|
# value the operator already holds — correct for it, and exactly why it is the wrong tool here.
|
||||||
|
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB and two dozen more are existing keys whose values must
|
||||||
|
# change, or they would go on naming the old layout forever while the new directory variables
|
||||||
|
# sat beside them unused.
|
||||||
|
#
|
||||||
|
# Move, never copy-and-hope.
|
||||||
|
# The data being relocated is the only copy — statistics, histories, the AI index, arr caches.
|
||||||
|
# Everything is moved and the source is gone afterwards, so there is no second location that
|
||||||
|
# might still be written to by something that missed the change.
|
||||||
|
#
|
||||||
|
# The conf rewrite is the last thing, and the riskiest thing.
|
||||||
|
# Until it happens the installation still works from the old layout. That ordering means an
|
||||||
|
# abort partway through leaves a system that runs, rather than one whose paths point at
|
||||||
|
# nothing.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -49,6 +85,25 @@
|
|||||||
# check costs nothing and the failure is silent otherwise.
|
# check costs nothing and the failure is silent otherwise.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# CONFIGURATION
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# This script reads conf to find the old locations and rewrites conf to record the new ones. It
|
||||||
|
# is the one script here whose purpose is to change these values rather than obey them.
|
||||||
|
#
|
||||||
|
# Read to locate what moves
|
||||||
|
# STATE_DIR, BANDWIDTH_LOG, AI_INDEX_DB, AI_MEMORY_FILE, AI_TOKEN_DB, ARR_CLEANUP_STATS,
|
||||||
|
# ARR_SYNC_BLOCKLIST, CORRUPTION_SCAN_STATE_FILE, LIDARR_CACHE_FILE, ZFS_REPORT_LOG and the
|
||||||
|
# rest of the per-script path keys — roughly two dozen in total.
|
||||||
|
#
|
||||||
|
# Written as the new roots
|
||||||
|
# DATA_DIR and the directories beneath it: DB_DIR, STATE_DIR, AI_DATA_DIR,
|
||||||
|
# CACHE_BACKUP_DIR, ARR_CACHE_BACKUP_DIR, CONF_CACHE_BACKUP_DIR, LOG_ARCHIVE_DIR.
|
||||||
|
#
|
||||||
|
# Every rewritten value is expressed as ${DB_DIR}/… rather than an absolute path, so a later
|
||||||
|
# storage-mode migration moves them again by changing one variable.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
Executable
+351
@@ -0,0 +1,351 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
# PURPOSE
|
||||||
|
# Put the containers this host has marked for fallback coverage onto the partner, so that the
|
||||||
|
# partner can actually start them during an outage — and take them off again on request.
|
||||||
|
#
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# fallback.sh covers a host by running `docker start <name>` on the partner. It never creates
|
||||||
|
# anything. So a name in FALLBACK_<me>_TIER* is a promise that only holds if the partner already
|
||||||
|
# has that container built. Measured 2026-08-23: all 12 of HOST1's covered containers were absent
|
||||||
|
# from HOST2, meaning every tier would have failed on the first real outage while the UI showed
|
||||||
|
# coverage as configured. This script is what closes that gap.
|
||||||
|
#
|
||||||
|
# Push and remove are separate, deliberate actions, never a side effect of saving the tier list.
|
||||||
|
# Editing coverage is a cheap config write; deploying a dozen containers onto another machine is
|
||||||
|
# not, and the two should not share a button.
|
||||||
|
#
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# Deployed, then verified STOPPED.
|
||||||
|
# A container built here and left running on the partner would be a second live instance of
|
||||||
|
# NextCloud, Gitea or PostgreSQL_Immich against the same data while this host is healthy.
|
||||||
|
# That is the danger_rsync_live_database_appdata failure with worse odds. Every deploy is
|
||||||
|
# followed by a stop and a re-inspect, and a container that will not stay stopped is an
|
||||||
|
# error, not a warning.
|
||||||
|
#
|
||||||
|
# Remove takes the container AND its appdata.
|
||||||
|
# Operator decision 2026-08-23: the button is explicit, so a removal should leave nothing
|
||||||
|
# behind to reason about later. The risk it accepts is narrow and worth naming — if the
|
||||||
|
# partner ever covered for us, ITS appdata is the newer copy and is what a handback rsyncs
|
||||||
|
# home. The NORMAL-state gate below closes the live-failover window; what it cannot see is
|
||||||
|
# a handback that partially failed and then returned to NORMAL, so the UI says so before
|
||||||
|
# asking.
|
||||||
|
#
|
||||||
|
# Two guards on the deletion itself: only paths under /mnt/*/appdata* are ever touched, and
|
||||||
|
# a bind of the appdata ROOT is refused outright — a container mounting /mnt/user/appdata
|
||||||
|
# would otherwise turn one removal into wiping every application on the partner.
|
||||||
|
#
|
||||||
|
# Refuses to run unless fallback state is NORMAL.
|
||||||
|
# Pushing or removing containers mid-outage edits the thing currently keeping services up.
|
||||||
|
#
|
||||||
|
# Coverage names are resolved to templates by <Name>, not by filename.
|
||||||
|
# my-Foo.xml routinely holds a container called something else. Matching on the filename
|
||||||
|
# silently pushes the wrong template, or nothing at all.
|
||||||
|
#
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# Only in NORMAL state. FALLBACK_STATE_FILE is read before anything is pushed or removed, and
|
||||||
|
# any other state refuses the action. A push during a live failover would deploy a second copy
|
||||||
|
# of a container the partner is currently running on our behalf; a remove would delete the one
|
||||||
|
# doing the covering.
|
||||||
|
#
|
||||||
|
# --status is exempt from that gate, because it only reports. Refusing to answer "what is
|
||||||
|
# deployed over there" during a failover would withhold the information precisely when it is
|
||||||
|
# most wanted.
|
||||||
|
#
|
||||||
|
# Every deploy is verified stopped, and a container that will not stay stopped is an error
|
||||||
|
# rather than a warning — see DESIGN PRINCIPLES. A second live instance against the same data
|
||||||
|
# is the failure this whole script exists inside.
|
||||||
|
#
|
||||||
|
# Push and remove are explicit modes with no default. Running the script with no flag does
|
||||||
|
# nothing; neither action can be reached by accident, and neither is a side effect of editing
|
||||||
|
# the tier list.
|
||||||
|
#
|
||||||
|
# --dry-run works in every mode and touches nothing on either host — no container is built,
|
||||||
|
# started, stopped or removed, and no template is written or deleted.
|
||||||
|
#
|
||||||
|
# Remove deletes the container's appdata on the partner as well. That is deliberate and is the
|
||||||
|
# most destructive thing here; the NORMAL-state gate above is what keeps it away from a
|
||||||
|
# partner that is mid-handback.
|
||||||
|
#
|
||||||
|
# CONFIGURATION
|
||||||
|
# master.conf
|
||||||
|
# FALLBACK_<HOST>_TIER1..N the covered container names — what --push deploys and --status
|
||||||
|
# reports on. This script reads that list; it never edits it.
|
||||||
|
#
|
||||||
|
# host*.conf
|
||||||
|
# FALLBACK_STATE_FILE overrides where fallback.sh's state is read from. Defaults to
|
||||||
|
# STATE_DIR/fallback_state.db. A missing file reads as NORMAL,
|
||||||
|
# which is the correct default on a host where fallback has never
|
||||||
|
# run.
|
||||||
|
#
|
||||||
|
# RUNTIME MODES
|
||||||
|
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
|
||||||
|
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
|
||||||
|
# coverage_deploy.sh --status report, per covered container, whether it exists there
|
||||||
|
# any mode supports --dry-run
|
||||||
|
#
|
||||||
|
# DEPENDS ON
|
||||||
|
# Plugin/<platform>/Partnership/containers.sh deploy_container_from_xml(), GPU transform
|
||||||
|
# FALLBACK_<me>_TIER1-4 the coverage list this acts on
|
||||||
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
||||||
|
|
||||||
|
SSH_TIMEOUT="${SSH_TIMEOUT:-15}"
|
||||||
|
MODE=""
|
||||||
|
DRY_RUN="${DRY_RUN:-false}"
|
||||||
|
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--push) MODE="push" ;;
|
||||||
|
--remove) MODE="remove" ;;
|
||||||
|
--status) MODE="status" ;;
|
||||||
|
--dry-run) DRY_RUN=true ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$MODE" ]]; then
|
||||||
|
error "No mode given — use --push, --remove or --status"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
detect_hosts
|
||||||
|
|
||||||
|
if [[ -z "$REMOTE_ID" || "$REMOTE_SERVER_NAME" == "unknown" ]]; then
|
||||||
|
error "No partner configured — nothing to push to"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Gate: only with fallback idle ─────────────────────────────────────────────────────────────
|
||||||
|
# Read rather than assumed. A missing state file means fallback has never run, which is idle
|
||||||
|
# enough; a file that says anything other than NORMAL means services are in motion right now.
|
||||||
|
FALLBACK_STATE_FILE="${FALLBACK_STATE_FILE:-${STATE_DIR}/fallback_state.db}"
|
||||||
|
_fb_state="NORMAL"
|
||||||
|
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||||
|
_fb_state=$(grep -m1 '^state=' "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||||
|
_fb_state="${_fb_state:-NORMAL}"
|
||||||
|
fi
|
||||||
|
if [[ "$_fb_state" != "NORMAL" && "$MODE" != "status" ]]; then
|
||||||
|
error "Fallback state is $_fb_state, not NORMAL — refusing to $MODE"
|
||||||
|
error "Changing what the partner holds while a failover is live edits the thing keeping services up."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── The coverage list ─────────────────────────────────────────────────────────────────────────
|
||||||
|
COVERED=()
|
||||||
|
for _t in 1 2 3 4; do
|
||||||
|
_var="FALLBACK_${MY_ID}_TIER${_t}[@]"
|
||||||
|
for _c in "${!_var}"; do
|
||||||
|
[[ -n "$_c" ]] && COVERED+=("$_c")
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${#COVERED[@]} -eq 0 ]]; then
|
||||||
|
warn "No containers are covered in FALLBACK_${MY_ID}_TIER1-4 — nothing to do"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "$ICON_FALLBACK Coverage: ${#COVERED[@]} container(s) for $REMOTE_SERVER_NAME to start during an outage"
|
||||||
|
|
||||||
|
resolve_remote_ip
|
||||||
|
MIRROR="$REMOTE_SERVER_NAME"
|
||||||
|
MIRROR_IP="$REMOTE_SERVER"
|
||||||
|
_key_var="${MY_ID}_SSH_KEY"
|
||||||
|
MIRROR_SSH_KEY="${!_key_var}"
|
||||||
|
|
||||||
|
if [[ ! -f "$MIRROR_SSH_KEY" ]]; then
|
||||||
|
error "SSH key $MIRROR_SSH_KEY not found — cannot reach $MIRROR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── name -> template ──────────────────────────────────────────────────────────────────────────
|
||||||
|
# Matched on the <Name> element. Filenames lie often enough that trusting them would push the
|
||||||
|
# wrong container without saying so.
|
||||||
|
xml_for_container() {
|
||||||
|
local want="$1" f n
|
||||||
|
for f in "$TEMPLATES_DIR"/*.xml; do
|
||||||
|
[[ -f "$f" ]] || continue
|
||||||
|
n=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$f")
|
||||||
|
[[ "$n" == "$want" ]] && { echo "$f"; return 0; }
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
remote_has_container() {
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker inspect $(printf '%q' "$1") >/dev/null 2>&1" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
remote_state_of() {
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker inspect -f '{{.State.Status}}' $(printf '%q' "$1") 2>/dev/null" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
OK=0; FAIL=0; SKIP=0
|
||||||
|
|
||||||
|
case "$MODE" in
|
||||||
|
|
||||||
|
status)
|
||||||
|
# Written as a cache as well as printed. The assistant's fallback_state block cannot afford an
|
||||||
|
# SSH round trip per container mid-question, so it reads this file and reports its AGE — a stale
|
||||||
|
# answer stated as stale is useful, stated as current it is the exact failure this feature
|
||||||
|
# exists to prevent.
|
||||||
|
_present="" _missing=""
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
if remote_has_container "$c"; then
|
||||||
|
_st=$(remote_state_of "$c")
|
||||||
|
printf ' %-28s on %s (%s)\n' "$c" "$MIRROR" "$_st"
|
||||||
|
_present+="\"$c\":\"${_st:-unknown}\","
|
||||||
|
OK=$((OK+1))
|
||||||
|
else
|
||||||
|
printf ' %-28s MISSING on %s — docker start would fail\n' "$c" "$MIRROR"
|
||||||
|
_missing+="\"$c\","
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "$ICON_FALLBACK Coverage present: $OK · missing: $FAIL"
|
||||||
|
|
||||||
|
mkdir -p "$VV_CACHE_ROOT/api" 2>/dev/null || mkdir -p /tmp/varaverk/api 2>/dev/null
|
||||||
|
_cache="${VV_CACHE_ROOT:-/tmp/varaverk}/api/fallback_presence.json"
|
||||||
|
# Written atomically — a half-written cache read mid-question would report containers as
|
||||||
|
# missing that are merely unparsed.
|
||||||
|
printf '{"present":{%s},"missing":[%s],"partner":"%s","checked":%s}\n' \
|
||||||
|
"${_present%,}" "${_missing%,}" "$MIRROR" "$(date +%s)" > "$_cache.tmp" \
|
||||||
|
&& mv -f "$_cache.tmp" "$_cache"
|
||||||
|
|
||||||
|
[[ "$FAIL" -gt 0 ]] && exit 2 || exit 0
|
||||||
|
;;
|
||||||
|
|
||||||
|
push)
|
||||||
|
# Networks first — a container whose network is absent is created and then cannot start,
|
||||||
|
# which is the failure that read as "auth 0/8, arr 0/5" during onboarding.
|
||||||
|
_nets=()
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
x=$(xml_for_container "$c") || continue
|
||||||
|
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$x" 2>/dev/null | head -1)
|
||||||
|
net="${net//[[:space:]]/}"
|
||||||
|
# br* is host hardware. wg* is a WireGuard-backed bridge whose meaning does NOT travel:
|
||||||
|
# recreating it on the partner as a plain bridge yields a network that exists, starts its
|
||||||
|
# containers, and routes their traffic OUTSIDE the tunnel. ChannelTube rides wg0 here.
|
||||||
|
case "$net" in
|
||||||
|
''|bridge|host|none|br[0-9]*) continue ;;
|
||||||
|
wg[0-9]*)
|
||||||
|
warn "$c uses $net — a WireGuard-backed network. NOT created on $MIRROR: a plain"
|
||||||
|
warn " bridge of the same name would route its traffic outside the tunnel. Build the"
|
||||||
|
warn " matching tunnel there first, or drop $c from coverage."
|
||||||
|
continue ;;
|
||||||
|
esac
|
||||||
|
_seen=false
|
||||||
|
for n in "${_nets[@]}"; do [[ "$n" == "$net" ]] && { _seen=true; break; }; done
|
||||||
|
[[ "$_seen" == false ]] && _nets+=("$net")
|
||||||
|
done
|
||||||
|
for net in "${_nets[@]}"; do
|
||||||
|
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" --format '{{.Driver}}' 2>/dev/null)
|
||||||
|
if [[ "$driver" != "bridge" ]]; then
|
||||||
|
warn "Network $net is '${driver:-absent}' here, not bridge — create it on $MIRROR by hand"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would ensure network $net on $MIRROR"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
||||||
|
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null \
|
||||||
|
&& log " network $net ready on $MIRROR" \
|
||||||
|
|| warn " could not ensure network $net on $MIRROR"
|
||||||
|
done
|
||||||
|
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
x=$(xml_for_container "$c") || {
|
||||||
|
warn "$c — no template in $TEMPLATES_DIR names it; skipped"
|
||||||
|
SKIP=$((SKIP+1)); continue
|
||||||
|
}
|
||||||
|
if ! deploy_container_from_xml "$x" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||||
|
error "$c — deploy failed"
|
||||||
|
FAIL=$((FAIL+1)); continue
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then OK=$((OK+1)); continue; fi
|
||||||
|
|
||||||
|
# Deployed containers must not run here. Stop, then re-inspect — a stop that did not take
|
||||||
|
# is the one outcome that silently duplicates a live service against shared data.
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker stop $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null
|
||||||
|
st=$(remote_state_of "$c")
|
||||||
|
if [[ "$st" == "running" ]]; then
|
||||||
|
error "$c is RUNNING on $MIRROR after deploy and would not stop — stop it there before continuing"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
else
|
||||||
|
log " $c deployed and ${st:-stopped} on $MIRROR ✅"
|
||||||
|
OK=$((OK+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "$ICON_FALLBACK Push complete — deployed $OK · failed $FAIL · skipped $SKIP"
|
||||||
|
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||||
|
;;
|
||||||
|
|
||||||
|
remove)
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
if ! remote_has_container "$c"; then
|
||||||
|
log " $c not on $MIRROR — nothing to remove"
|
||||||
|
SKIP=$((SKIP+1)); continue
|
||||||
|
fi
|
||||||
|
# Binds are read BEFORE the container goes — once it is removed there is nothing left to
|
||||||
|
# enumerate, and a path list gathered afterwards would silently be empty.
|
||||||
|
_binds=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' $(printf '%q' "$c") 2>/dev/null \
|
||||||
|
| awk -F: '{print \$1}'" 2>/dev/null)
|
||||||
|
|
||||||
|
_wipe=()
|
||||||
|
while IFS= read -r _p; do
|
||||||
|
[[ -z "$_p" ]] && continue
|
||||||
|
# Only appdata, and never an appdata root. /mnt/user/appdata as a bind would make one
|
||||||
|
# container removal delete every application on the partner.
|
||||||
|
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/.+ ]] || {
|
||||||
|
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/?$ ]] && \
|
||||||
|
warn " $c binds the appdata ROOT ($_p) — refusing to delete it"
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_wipe+=("$_p")
|
||||||
|
done <<< "$_binds"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would stop and remove $c on $MIRROR"
|
||||||
|
for _p in "${_wipe[@]}"; do warn " DRY RUN — would delete appdata $_p on $MIRROR"; done
|
||||||
|
OK=$((OK+1)); continue
|
||||||
|
fi
|
||||||
|
x=$(xml_for_container "$c") && xml_name=$(basename "$x") || xml_name=""
|
||||||
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker stop $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||||
|
docker rm $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||||
|
${xml_name:+rm -f ${TEMPLATES_DIR}/$(printf '%q' "$xml_name");} \
|
||||||
|
! docker inspect $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null; then
|
||||||
|
log " $c removed from $MIRROR ✅"
|
||||||
|
for _p in "${_wipe[@]}"; do
|
||||||
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"rm -rf -- $(printf '%q' "$_p") && ! [ -e $(printf '%q' "$_p") ]" 2>/dev/null; then
|
||||||
|
log " appdata deleted on $MIRROR: $_p"
|
||||||
|
else
|
||||||
|
warn " could not delete appdata on $MIRROR: $_p"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
OK=$((OK+1))
|
||||||
|
else
|
||||||
|
error "$c — removal failed or it still exists on $MIRROR"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "$ICON_FALLBACK Remove complete — removed $OK · failed $FAIL · skipped $SKIP"
|
||||||
|
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
+85
-4
@@ -314,7 +314,51 @@ if [[ "$EUID" -ne 0 ]]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt).
|
|
||||||
|
# ── Persistent dry-run ────────────────────────────────────────────────────────────────────────
|
||||||
|
# Conf-driven, not argument-driven, deliberately. array_started.sh launches every entry as a bare
|
||||||
|
# `bash script.sh &` with no arguments, so a --dry-run typed at a terminal survives exactly until
|
||||||
|
# the next array start — and then the LIVE daemon comes up in its place, silently, which is the
|
||||||
|
# one transition nobody would be watching for.
|
||||||
|
#
|
||||||
|
# Setting it here means the mode is a property of the install rather than of how the process
|
||||||
|
# happened to be started: array start, the Fallback tab's button, and a hand-run all agree.
|
||||||
|
#
|
||||||
|
# OR, never override: --dry-run on the command line still wins over a conf that says false, so an
|
||||||
|
# ad-hoc preview against a live install needs no conf edit.
|
||||||
|
if [[ "${FALLBACK_DRY_RUN:-false}" == "true" ]]; then
|
||||||
|
DRY_RUN=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Persistent log ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# array_started.sh launches every entry as a bare `bash script.sh &` with no redirection, so this
|
||||||
|
# daemon's output has never been captured anywhere: /var/log/varaverk has a directory for every
|
||||||
|
# other script family and none for Fallback. A month of dry-run observation would have persisted
|
||||||
|
# nothing at all.
|
||||||
|
#
|
||||||
|
# /var/log is a 128 MB tmpfs on Unraid — RAM, and cleared on reboot — so the log goes to
|
||||||
|
# LOG_ARCHIVE_DIR, which follows DATA_DIR onto real storage.
|
||||||
|
#
|
||||||
|
# Only when stdout is not a terminal. Run by hand, output still goes to the terminal exactly as
|
||||||
|
# before; run by array_started or the Fallback tab's button, it lands in the file. A plain append
|
||||||
|
# redirect rather than `tee` through process substitution: no extra child to outlive, and nothing
|
||||||
|
# for the shutdown trap to race.
|
||||||
|
FALLBACK_LOG="${LOG_ARCHIVE_DIR:-${DATA_DIR:-/tmp}/logs}/fallback.log"
|
||||||
|
if [[ ! -t 1 ]]; then
|
||||||
|
mkdir -p "$(dirname "$FALLBACK_LOG")" 2>/dev/null
|
||||||
|
# One rotation, sized rather than line-counted — the whole point of this log is a long run,
|
||||||
|
# and _orch_trim_log()'s 1000-line cap would discard weeks of it. Event-only output (no
|
||||||
|
# --log) is a few lines per incident, so this holds years; --log fills it in about a fortnight
|
||||||
|
# and then keeps the most recent fortnight plus the one before it.
|
||||||
|
_fb_max=$(( ${FALLBACK_LOG_MAX_MB:-5} * 1048576 ))
|
||||||
|
if [[ -f "$FALLBACK_LOG" ]] && (( $(stat -c %s "$FALLBACK_LOG" 2>/dev/null || echo 0) > _fb_max )); then
|
||||||
|
mv -f "$FALLBACK_LOG" "${FALLBACK_LOG}.1" 2>/dev/null
|
||||||
|
fi
|
||||||
|
exec >> "$FALLBACK_LOG" 2>&1
|
||||||
|
echo ""
|
||||||
|
echo "═══ fallback.sh started $(date '+%Y-%m-%d %H:%M:%S') — dry_run=${DRY_RUN} pid=$$ ═══"
|
||||||
|
fi
|
||||||
|
# FALLBACK_ENABLED gate — exits cleanly when disabled.
|
||||||
# Fail-closed: anything that isn't exactly "true" disables fallback. Matching only the
|
# Fail-closed: anything that isn't exactly "true" disables fallback. Matching only the
|
||||||
# literal "false" would let a typo ("no", "0", "FALSE") hand this script DDNS authority
|
# literal "false" would let a typo ("no", "0", "FALSE") hand this script DDNS authority
|
||||||
# and cross-server container control on a toggle nobody meant to set.
|
# and cross-server container control on a toggle nobody meant to set.
|
||||||
@@ -368,6 +412,22 @@ state_set() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
state_init() {
|
state_init() {
|
||||||
|
# A dry run must not leave the host believing it failed over. state_set() writes
|
||||||
|
# unconditionally, and this file survives reboots and is what the real daemon — and the
|
||||||
|
# Monitor and Fallback cards — read to decide what is happening. A --dry-run walk through
|
||||||
|
# FAILOVER would have written state=FALLBACK, the tier flags and the strike counter into it
|
||||||
|
# for real, and nothing would have put them back.
|
||||||
|
#
|
||||||
|
# Copied rather than merely redirected, so the preview still starts from the live state and
|
||||||
|
# can advance through tiers exactly as a real run would. The copy lands in the RAM cache and
|
||||||
|
# dies with the reboot.
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
local live="$FALLBACK_STATE_FILE"
|
||||||
|
FALLBACK_STATE_FILE="${VV_CACHE_ROOT:-/tmp/varaverk}/fallback_state.dryrun.$$"
|
||||||
|
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||||
|
if [[ -f "$live" ]]; then cp -f "$live" "$FALLBACK_STATE_FILE"; else : > "$FALLBACK_STATE_FILE"; fi
|
||||||
|
warn "DRY RUN — state writes redirected to $FALLBACK_STATE_FILE (live state untouched)"
|
||||||
|
fi
|
||||||
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
|
||||||
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
[[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
|
||||||
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
[[ -z "$(state_get state)" ]] && state_set state "NORMAL"
|
||||||
@@ -759,8 +819,16 @@ run_handback() {
|
|||||||
for job in "${jobs[@]}"; do
|
for job in "${jobs[@]}"; do
|
||||||
[[ -z "$job" ]] && continue
|
[[ -z "$job" ]] && continue
|
||||||
log "Syncing: $job"
|
log "Syncing: $job"
|
||||||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
# if/else, not A && B || C. In the shorthand a REAL run whose rsync exits
|
||||||
|| warn "DRY RUN — would rsync: $job"
|
# non-zero falls through to the || branch and logs "DRY RUN — would rsync",
|
||||||
|
# so a failed Tier writeback reported itself as a preview and the real
|
||||||
|
# failure went unsaid. The Tier 1 block below always had this right.
|
||||||
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||||
|
|| error "Tier $tier writeback FAILED: $job"
|
||||||
|
else
|
||||||
|
warn "DRY RUN — would rsync: $job"
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||||||
@@ -904,6 +972,19 @@ echo " $ICON_NET Remote IP: $REMOTE_SERVER"
|
|||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
FALLBACK_RUNNING=true
|
FALLBACK_RUNNING=true
|
||||||
|
|
||||||
|
# The dry-run state copy is per-PID and would otherwise accumulate one file per preview run.
|
||||||
|
# EXIT as well as the signals, because a dry run is usually ended with Ctrl-C or --stop but can
|
||||||
|
# also just fall out of the loop.
|
||||||
|
dryrun_state_cleanup() {
|
||||||
|
[[ "$DRY_RUN" == true && "$FALLBACK_STATE_FILE" == *".dryrun."* ]] && rm -f "$FALLBACK_STATE_FILE"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
# Must call _release_all_locks too. acquire_lock() registers its own EXIT trap, and bash keeps
|
||||||
|
# exactly one per signal — a bare `trap ... EXIT` here silently replaced it and orphaned
|
||||||
|
# fallback.lock, which is the precise failure the _LOCK_FILES registry in common.sh was built to
|
||||||
|
# stop. The signal trap only needs `exit 0`; that fires EXIT, which does both jobs.
|
||||||
|
trap 'dryrun_state_cleanup; _release_all_locks' EXIT
|
||||||
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
|
||||||
SIGTERM SIGINT
|
SIGTERM SIGINT
|
||||||
|
|
||||||
@@ -1185,4 +1266,4 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
|||||||
sleep "$FALLBACK_CHECK_INTERVAL" &
|
sleep "$FALLBACK_CHECK_INTERVAL" &
|
||||||
wait $!
|
wait $!
|
||||||
|
|
||||||
done
|
done
|
||||||
|
|||||||
@@ -118,6 +118,11 @@
|
|||||||
# fallback_test.sh --log
|
# fallback_test.sh --log
|
||||||
# Verbose output on every check in every phase.
|
# Verbose output on every check in every phase.
|
||||||
#
|
#
|
||||||
|
# fallback_test.sh --stop
|
||||||
|
# Stop a running test. SIGTERM only — never SIGKILL, because only this script's EXIT
|
||||||
|
# trap removes the iptables DROP rule it installed. Also sweeps a rule stranded by an
|
||||||
|
# earlier SIGKILL or power cut.
|
||||||
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
@@ -126,6 +131,58 @@ source "$SCRIPT_DIR/../load_config.sh"
|
|||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
|
# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
|
||||||
|
#
|
||||||
|
# SIGTERM ONLY, and deliberately no SIGKILL escalation — the opposite of fallback.sh --stop.
|
||||||
|
# A running test holds an iptables DROP rule against the partner, and the only thing that removes
|
||||||
|
# it is this script's own EXIT trap. SIGKILL does not run traps, so force-killing a test strands
|
||||||
|
# the rule: the partner stays invisible, fallback.sh reads that as a permanent outage and holds
|
||||||
|
# FALLBACK indefinitely. A test that will not die is a worse outcome than a test still running,
|
||||||
|
# so this reports the stranded rule and the command to clear it rather than causing one.
|
||||||
|
if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
|
||||||
|
LOCKFILE="${LOCK_DIR}/fallback_test.lock"
|
||||||
|
if [[ ! -f "$LOCKFILE" ]]; then
|
||||||
|
log "No fallback_test.sh lock found — not running"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
lock_content=$(cat "$LOCKFILE" 2>/dev/null)
|
||||||
|
target_pid="${lock_content%%:*}"
|
||||||
|
if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
|
||||||
|
warn "Stale lock — fallback_test.sh not running (PID ${target_pid:-unknown} gone) — clearing"
|
||||||
|
rm -f "$LOCKFILE"
|
||||||
|
# A stale lock is exactly the SIGKILL/power-cut case, so the rule may still be in place.
|
||||||
|
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||||
|
warn "Stranded iptables DROP rule found for $REMOTE_SERVER — removing"
|
||||||
|
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||||
|
&& warn "Stranded rule removed — remote connectivity restored ✅" \
|
||||||
|
|| error "Could not remove stranded rule — run: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
warn "Stopping fallback_test.sh (PID $target_pid) — SIGTERM so its trap clears the iptables rule..."
|
||||||
|
kill -TERM "$target_pid" 2>/dev/null || true
|
||||||
|
waited=0
|
||||||
|
while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 30 ]]; do
|
||||||
|
sleep 1
|
||||||
|
(( waited++ )) || true
|
||||||
|
done
|
||||||
|
if kill -0 "$target_pid" 2>/dev/null; then
|
||||||
|
error "fallback_test.sh (PID $target_pid) did not exit within 30s"
|
||||||
|
error "NOT force-killing — SIGKILL would strand the iptables DROP rule on $REMOTE_SERVER"
|
||||||
|
error "Wait, or clear manually: iptables -D OUTPUT -d $REMOTE_SERVER -j DROP"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
warn "Stopped: fallback_test.sh (PID $target_pid) ✅"
|
||||||
|
if iptables -C OUTPUT -d "${REMOTE_SERVER:-0.0.0.0}" -j DROP 2>/dev/null; then
|
||||||
|
error "iptables DROP rule for $REMOTE_SERVER survived the stop — removing"
|
||||||
|
iptables -D OUTPUT -d "$REMOTE_SERVER" -j DROP 2>/dev/null \
|
||||||
|
&& warn "Rule removed ✅" || error "Could not remove — run it by hand"
|
||||||
|
else
|
||||||
|
log "No iptables DROP rule remains for $REMOTE_SERVER ✅"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
FALLBACK_SCRIPT="$SCRIPT_DIR/fallback.sh"
|
||||||
DOCKER_TIMEOUT=15
|
DOCKER_TIMEOUT=15
|
||||||
|
|
||||||
@@ -325,6 +382,27 @@ else
|
|||||||
warn "No state file found — assuming NORMAL (first run)"
|
warn "No state file found — assuming NORMAL (first run)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# fallback.sh must actually be RUNNING, not merely enabled
|
||||||
|
#
|
||||||
|
# Every phase after this one waits for the daemon to change state. FALLBACK_ENABLED=true says
|
||||||
|
# it is allowed to run; it does not say array_started.sh launched it, or that it is still alive.
|
||||||
|
# Without this the test passes pre-flight, drops a real iptables rule on the partner, waits
|
||||||
|
# FALLBACK_TEST_BLOCK_WAIT for a transition nothing is there to make, and fails Phase 3 blaming
|
||||||
|
# fallback detection. Only the EXIT trap gets connectivity back.
|
||||||
|
#
|
||||||
|
# In --dry-run nothing is blocked and nothing is waited on, so a dead daemon is worth saying but
|
||||||
|
# not worth aborting for — the walkthrough still shows the operator the shape of the run.
|
||||||
|
if pgrep -f "Fallback/fallback\.sh" >/dev/null 2>&1; then
|
||||||
|
log "fallback.sh daemon is running"
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "fallback.sh is NOT running — a real test would abort here"
|
||||||
|
else
|
||||||
|
error "fallback.sh is not running — nothing would detect the outage this test creates"
|
||||||
|
error "Start it with array_started.sh, or run with --dry-run to walk the phases"
|
||||||
|
phase_fail "Pre-flight"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
# Tier 1 containers configured
|
# Tier 1 containers configured
|
||||||
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
if [[ ${#TIER1_CONTAINERS[@]} -eq 0 ]]; then
|
||||||
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
error "No Tier 1 containers configured for $MY_ID → $REMOTE_ID"
|
||||||
|
|||||||
Executable → Regular
+46
-10
@@ -122,6 +122,7 @@ source "$SCRIPT_DIR/../load_config.sh"
|
|||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
SSH_TIMEOUT=15
|
SSH_TIMEOUT=15
|
||||||
|
BACKUP_VERIFY_MD5_TIMEOUT_MAX="${BACKUP_VERIFY_MD5_TIMEOUT_MAX:-600}"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Setup ━━━
|
# ━━━ Setup ━━━
|
||||||
@@ -228,6 +229,7 @@ TOTAL_CHECKED=0
|
|||||||
TOTAL_MATCH=0
|
TOTAL_MATCH=0
|
||||||
TOTAL_MISMATCH=0
|
TOTAL_MISMATCH=0
|
||||||
TOTAL_MISSING=0
|
TOTAL_MISSING=0
|
||||||
|
TOTAL_UNVERIFIED=0
|
||||||
SHARES_WITH_ISSUES=()
|
SHARES_WITH_ISSUES=()
|
||||||
|
|
||||||
for share in "${VERIFY_SHARES[@]}"; do
|
for share in "${VERIFY_SHARES[@]}"; do
|
||||||
@@ -265,6 +267,7 @@ for share in "${VERIFY_SHARES[@]}"; do
|
|||||||
SHARE_MATCH=0
|
SHARE_MATCH=0
|
||||||
SHARE_MISMATCH=0
|
SHARE_MISMATCH=0
|
||||||
SHARE_MISSING=0
|
SHARE_MISSING=0
|
||||||
|
SHARE_UNVERIFIED=0
|
||||||
|
|
||||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||||
[[ -z "$local_file" ]] && continue
|
[[ -z "$local_file" ]] && continue
|
||||||
@@ -276,19 +279,49 @@ for share in "${VERIFY_SHARES[@]}"; do
|
|||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Remote checksum via SSH — timeout protected
|
# The path is interpolated into a remote shell command, so it must be escaped for
|
||||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
# reuse as one word. A bare '$local_file' inside single quotes breaks on the first
|
||||||
|
# apostrophe — "Frieren - Beyond Journey's End" ended the quote early, md5sum fell
|
||||||
|
# back to reading stdin, and the empty-input hash d41d8cd9... was reported as a
|
||||||
|
# MISMATCH against a file that is byte-identical on the remote.
|
||||||
|
printf -v remote_q '%q' "$local_file"
|
||||||
|
|
||||||
|
# Existence and content are separate questions. Asking them together means a slow
|
||||||
|
# checksum is indistinguishable from an absent file.
|
||||||
|
remote_exists=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
-o StrictHostKeyChecking=no \
|
-o StrictHostKeyChecking=no \
|
||||||
root@"$REMOTE_SERVER" \
|
root@"$REMOTE_SERVER" \
|
||||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
"test -f $remote_q && echo yes" 2>/dev/null </dev/null)
|
||||||
|
|
||||||
(( TOTAL_CHECKED++ ))
|
(( TOTAL_CHECKED++ ))
|
||||||
|
|
||||||
if [[ -z "$remote_md5" ]]; then
|
if [[ "$remote_exists" != "yes" ]]; then
|
||||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||||
(( SHARE_MISSING++ ))
|
(( SHARE_MISSING++ ))
|
||||||
(( TOTAL_MISSING++ ))
|
(( TOTAL_MISSING++ ))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# md5sum of a multi-GB file cannot finish inside a connect-sized timeout. Budget by
|
||||||
|
# size — a 5.9GB file needs ~30s and was being killed at 15s, then counted MISSING
|
||||||
|
# even though it was present and correct.
|
||||||
|
local_size=$(stat -c%s "$local_file" 2>/dev/null || echo 0)
|
||||||
|
md5_timeout=$(( local_size / 52428800 + SSH_TIMEOUT ))
|
||||||
|
(( md5_timeout > BACKUP_VERIFY_MD5_TIMEOUT_MAX )) && md5_timeout=$BACKUP_VERIFY_MD5_TIMEOUT_MAX
|
||||||
|
|
||||||
|
remote_md5=$(timeout "$md5_timeout" ssh -i "$SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o StrictHostKeyChecking=no \
|
||||||
|
root@"$REMOTE_SERVER" \
|
||||||
|
"md5sum $remote_q 2>/dev/null | awk '{print \$1}'" 2>/dev/null </dev/null)
|
||||||
|
|
||||||
|
if [[ -z "$remote_md5" ]]; then
|
||||||
|
# Present but unreadable within budget. Reporting this as a mismatch or a miss
|
||||||
|
# would be a claim the run did not earn.
|
||||||
|
warn "$ICON_WARN UNVERIFIED (checksum timed out after ${md5_timeout}s): $(basename "$local_file")"
|
||||||
|
(( SHARE_UNVERIFIED++ ))
|
||||||
|
(( TOTAL_UNVERIFIED++ ))
|
||||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||||
log "MATCH: $(basename "$local_file")"
|
log "MATCH: $(basename "$local_file")"
|
||||||
(( SHARE_MATCH++ ))
|
(( SHARE_MATCH++ ))
|
||||||
@@ -303,8 +336,8 @@ for share in "${VERIFY_SHARES[@]}"; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
# Per-share result — only visible if issues found
|
# Per-share result — only visible if issues found
|
||||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 || "$SHARE_UNVERIFIED" -gt 0 ]]; then
|
||||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH unverified: $SHARE_UNVERIFIED"
|
||||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||||
else
|
else
|
||||||
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
echo "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||||
@@ -325,10 +358,11 @@ echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
|||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 || "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||||
warn "Missing: $TOTAL_MISSING"
|
warn "Missing: $TOTAL_MISSING"
|
||||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||||
|
[[ "$TOTAL_UNVERIFIED" -gt 0 ]] && warn "Unverified: $TOTAL_UNVERIFIED (present, checksum timed out)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
@@ -337,6 +371,8 @@ elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
|||||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||||
"Backup Verify" "warning"
|
"Backup Verify" "warning"
|
||||||
|
elif [[ "$TOTAL_UNVERIFIED" -gt 0 ]]; then
|
||||||
|
warn "Status: $TOTAL_MATCH verified, $TOTAL_UNVERIFIED could not be checksummed in time — NOT a clean run"
|
||||||
else
|
else
|
||||||
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
echo "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||||
fi
|
fi
|
||||||
|
|||||||
Executable → Regular
+17
-17
@@ -18,6 +18,22 @@
|
|||||||
# WebGUI slowdowns or timeouts under load.
|
# WebGUI slowdowns or timeouts under load.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Each run snapshots:
|
||||||
|
# inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit.
|
||||||
|
# Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT.
|
||||||
|
# php-fpm: active workers vs PHP_MAX_CHILDREN limit.
|
||||||
|
# Warns above PHP_FPM_WARN_PCT.
|
||||||
|
#
|
||||||
|
# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days):
|
||||||
|
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|
|
||||||
|
# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||||||
|
# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts
|
||||||
|
# warnings over the week to show trend severity.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -36,22 +52,6 @@
|
|||||||
# signal — routine snapshots below the threshold produce nothing.
|
# signal — routine snapshots below the threshold produce nothing.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
|
||||||
# ==============================================================================================
|
|
||||||
#
|
|
||||||
# Each run snapshots:
|
|
||||||
# inotify: instances in use vs INOTIFY_MAX_INSTANCES kernel limit.
|
|
||||||
# Top 5 consumers by instance count. Warns above INOTIFY_WARN_PCT.
|
|
||||||
# php-fpm: active workers vs PHP_MAX_CHILDREN limit.
|
|
||||||
# Warns above PHP_FPM_WARN_PCT.
|
|
||||||
#
|
|
||||||
# Log line format (one per run, trimmed to TUNING_LOG_RETENTION days):
|
|
||||||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|
|
|
||||||
# PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
|
||||||
# INOTIFY_WARN and PHPFPM_WARN are 1/0 flags. weekly_health_digest.sh counts
|
|
||||||
# warnings over the week to show trend severity.
|
|
||||||
#
|
|
||||||
# ==============================================================================================
|
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -259,4 +259,4 @@ echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_
|
|||||||
>> "$TUNING_MONITOR_LOG"
|
>> "$TUNING_MONITOR_LOG"
|
||||||
|
|
||||||
echo "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
|
echo "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
|
||||||
log "Entry: ${DATE}|${TIME}|${INOTIFY_USED}/${INOTIFY_LIMIT}(${INOTIFY_PCT}%,warn=${INOTIFY_WARN})|${PHPFPM_ACTIVE}/${PHPFPM_MAX}(${PHPFPM_PCT}%,warn=${PHPFPM_WARN})"
|
log "Entry: ${DATE}|${TIME}|${INOTIFY_USED}/${INOTIFY_LIMIT}(${INOTIFY_PCT}%,warn=${INOTIFY_WARN})|${PHPFPM_ACTIVE}/${PHPFPM_MAX}(${PHPFPM_PCT}%,warn=${PHPFPM_WARN})"
|
||||||
|
|||||||
@@ -20,6 +20,42 @@
|
|||||||
# decision to notify is the exit code rather than this script parsing the text it just printed.
|
# decision to notify is the exit code rather than this script parsing the text it just printed.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Silence is the normal output.
|
||||||
|
# A report that always says something is a report nobody reads. A perfect week prints nothing
|
||||||
|
# and notifies nothing, so anything that does appear in the Sunday report is worth the glance.
|
||||||
|
#
|
||||||
|
# The exit code is the decision, not the text.
|
||||||
|
# uptime_probe.php --report exits 1 when it has something to say and 0 when it does not. This
|
||||||
|
# script never parses the output it just printed to work out whether to notify — a report whose
|
||||||
|
# wording changed would otherwise silently stop notifying.
|
||||||
|
#
|
||||||
|
# It reads; it never probes.
|
||||||
|
# The measurements are already taken, once a minute, by Tools/uptime_probe.sh. Re-probing at
|
||||||
|
# report time would describe Sunday morning rather than the week being reported on.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Read-only. Reads the stored history and prints; records nothing, and cannot alter the data it
|
||||||
|
# is reporting on.
|
||||||
|
#
|
||||||
|
# UPTIME_PROBE_ENABLED gates the whole run — with the probe off there is no history worth
|
||||||
|
# reporting, and this says nothing rather than reporting an empty week as a perfect one.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# RUNTIME MODES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# uptime_report.sh the weekly read. Silent when every domain was 100%.
|
||||||
|
#
|
||||||
|
# Called from COFFEE_REPORT_SCRIPTS; takes no arguments and has no other mode. For live figures
|
||||||
|
# or a per-domain table, use Tools/uptime_probe.sh --status.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
Executable → Regular
+19
-19
@@ -14,24 +14,6 @@
|
|||||||
# into a single digest. Reads only — writes nothing, changes nothing.
|
# into a single digest. Reads only — writes nothing, changes nothing.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
|
||||||
# ==============================================================================================
|
|
||||||
#
|
|
||||||
# Aggregator, Not Generator
|
|
||||||
# This script reads state files that other scripts maintain. It never produces
|
|
||||||
# health data itself — it only presents what is already there. Each source
|
|
||||||
# script remains responsible for its own state; this script is the envelope.
|
|
||||||
#
|
|
||||||
# Profile-Driven Notification
|
|
||||||
# The cron schedule never changes. The DIGEST_PROFILE in master.conf controls
|
|
||||||
# when notifications actually send — switching from daily noise to weekly
|
|
||||||
# summaries is a one-line conf change, not a cron edit.
|
|
||||||
#
|
|
||||||
# Read-Only, No Side Effects
|
|
||||||
# Writes nothing, changes nothing, triggers nothing. Safe to run at any time
|
|
||||||
# for a health snapshot without affecting any running service or state file.
|
|
||||||
#
|
|
||||||
# ==============================================================================================
|
|
||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -58,6 +40,24 @@
|
|||||||
# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
|
# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Aggregator, Not Generator
|
||||||
|
# This script reads state files that other scripts maintain. It never produces
|
||||||
|
# health data itself — it only presents what is already there. Each source
|
||||||
|
# script remains responsible for its own state; this script is the envelope.
|
||||||
|
#
|
||||||
|
# Profile-Driven Notification
|
||||||
|
# The cron schedule never changes. The DIGEST_PROFILE in master.conf controls
|
||||||
|
# when notifications actually send — switching from daily noise to weekly
|
||||||
|
# summaries is a one-line conf change, not a cron edit.
|
||||||
|
#
|
||||||
|
# Read-Only, No Side Effects
|
||||||
|
# Writes nothing, changes nothing, triggers nothing. Safe to run at any time
|
||||||
|
# for a health snapshot without affecting any running service or state file.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -394,4 +394,4 @@ if [[ "$DRY_RUN" == true ]]; then
|
|||||||
elif [[ "$SHOULD_SEND" == true ]]; then
|
elif [[ "$SHOULD_SEND" == true ]]; then
|
||||||
notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
|
notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
|
||||||
echo "Digest sent"
|
echo "Digest sent"
|
||||||
fi
|
fi
|
||||||
|
|||||||
Executable → Regular
+19
-19
@@ -15,6 +15,24 @@
|
|||||||
# comparison. In --dry-run mode, console only — nothing written to the log.
|
# comparison. In --dry-run mode, console only — nothing written to the log.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Five report sections (each skips gracefully if its data source is unavailable):
|
||||||
|
#
|
||||||
|
# ZFS pool health — status, state, errors per pool. Pools in
|
||||||
|
# ZFS_REPORT_IGNORE_POOLS excluded from the report
|
||||||
|
# (still fully monitored by unRAID — report-only exclusion).
|
||||||
|
# ARC statistics — current ARC vs max, metadata pressure, hit rate.
|
||||||
|
# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT, or if
|
||||||
|
# ARC headroom (max - current) drops below ZFS_REPORT_ARC_FREE_WARN_GB.
|
||||||
|
# Memory status — total, free, available RAM (informational only — see note below).
|
||||||
|
# Warns if available < ZFS_REPORT_AVAIL_WARN_GB.
|
||||||
|
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
|
||||||
|
# Useful for spotting containers approaching watchdog limits.
|
||||||
|
# Kernel pressure — vmstat snapshot (3 samples).
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -42,24 +60,6 @@
|
|||||||
# reclaimable cache) is still checked separately as a true system-pressure signal.
|
# reclaimable cache) is still checked separately as a true system-pressure signal.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
|
||||||
# ==============================================================================================
|
|
||||||
#
|
|
||||||
# Five report sections (each skips gracefully if its data source is unavailable):
|
|
||||||
#
|
|
||||||
# ZFS pool health — status, state, errors per pool. Pools in
|
|
||||||
# ZFS_REPORT_IGNORE_POOLS excluded from the report
|
|
||||||
# (still fully monitored by unRAID — report-only exclusion).
|
|
||||||
# ARC statistics — current ARC vs max, metadata pressure, hit rate.
|
|
||||||
# Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT, or if
|
|
||||||
# ARC headroom (max - current) drops below ZFS_REPORT_ARC_FREE_WARN_GB.
|
|
||||||
# Memory status — total, free, available RAM (informational only — see note below).
|
|
||||||
# Warns if available < ZFS_REPORT_AVAIL_WARN_GB.
|
|
||||||
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
|
|
||||||
# Useful for spotting containers approaching watchdog limits.
|
|
||||||
# Kernel pressure — vmstat snapshot (3 samples).
|
|
||||||
#
|
|
||||||
# ==============================================================================================
|
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -390,4 +390,4 @@ fi
|
|||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
|
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -266,16 +266,8 @@ echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
|||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# The configured list is the denominator — a script the conf names but that never launched is
|
||||||
warn "DRY RUN — no scripts launched"
|
# skipped, not absent, and only shows up if something counts it.
|
||||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
JOB_COUNT="${#ARRAY_START_SCRIPTS[@]}"
|
||||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
orchestrator_summary "ARRAY START" "$START" "Array Start"
|
||||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
exit $?
|
||||||
"Array Start" "warning"
|
|
||||||
else
|
|
||||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
@@ -190,18 +190,6 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
JOB_COUNT="$STEP"
|
||||||
warn "DRY RUN — no changes made"
|
orchestrator_summary "ARRAY STOP" "$START" "Array Stop"
|
||||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
exit $?
|
||||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
|
||||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
|
||||||
"Array Stop" "normal"
|
|
||||||
else
|
|
||||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
|
||||||
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
|
||||||
"Array Stop" "warning"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -244,9 +244,28 @@ if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
|||||||
PARTNER_DRY=""
|
PARTNER_DRY=""
|
||||||
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
||||||
|
|
||||||
|
# A successful rsync proves the partner answered; a failed one is evidence it did not. But
|
||||||
|
# rsync being switched off is neither — and it used to be read as "unseen", so the offline
|
||||||
|
# counter climbed every 30 minutes toward the 30-day auto-offboard on a partnership whose
|
||||||
|
# only fault was that RSYNC_ENABLED=false. That is how a deliberately paused sync ends up
|
||||||
|
# dismantling the partnership it was paused for. With no rsync attempt there is nothing to
|
||||||
|
# report, so the check runs without touching the counter either way.
|
||||||
|
# Tier 2 counts as "switched off" here exactly as much as Tier 1 does. The guard below used
|
||||||
|
# to test RSYNC_ENABLED alone, but it is CRITICAL_RSYNC_ENABLED that governs whether this
|
||||||
|
# orchestrator attempts an rsync at all — so with Tier 1 open and Tier 2 closed, no transfer
|
||||||
|
# was attempted, RSYNC_OK stayed false, and the run fell through to --remote-unseen and
|
||||||
|
# incremented the counter every 30 minutes against a partner that was answering fine.
|
||||||
|
#
|
||||||
|
# Onboard Step 1d now leaves precisely that posture on purpose — Tier 1 open so provisioning
|
||||||
|
# can run, every Tier 2 gate closed so nothing is scheduled. A freshly onboarded, perfectly
|
||||||
|
# healthy partnership would have auto-offboarded itself 30 days later.
|
||||||
if [[ "$RSYNC_OK" == true ]]; then
|
if [[ "$RSYNC_OK" == true ]]; then
|
||||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||||
--check --remote-seen $PARTNER_DRY
|
--check --remote-seen $PARTNER_DRY
|
||||||
|
elif [[ "${RSYNC_ENABLED:-false}" != true || "${CRITICAL_RSYNC_ENABLED:-false}" != true ]]; then
|
||||||
|
echo "Critical rsync gated off — partnership check runs, offline counter untouched"
|
||||||
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||||
|
--check $PARTNER_DRY
|
||||||
else
|
else
|
||||||
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
||||||
--check --remote-unseen $PARTNER_DRY
|
--check --remote-unseen $PARTNER_DRY
|
||||||
@@ -261,23 +280,7 @@ fi
|
|||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
DURATION=$(format_duration $(( END - START )))
|
DURATION=$(format_duration $(( END - START )))
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
# Standard ending, quiet mode — 30-min cadence, so a healthy cycle stays one line.
|
||||||
|
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||||
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
orchestrator_summary "CRITICAL SYNC" "$START" "Critical Sync" quiet
|
||||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
exit $?
|
||||||
echo ""
|
|
||||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Duration: $DURATION"
|
|
||||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
|
||||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
|
||||||
"Critical Sync" "warning"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
@@ -354,11 +354,6 @@ WINDOW_END=$(date +%s)
|
|||||||
# ━━━ Summary ━━━
|
# ━━━ Summary ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||||
for entry in "${SHARE_TIMES[@]}"; do
|
for entry in "${SHARE_TIMES[@]}"; do
|
||||||
@@ -382,16 +377,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
echo ""
|
echo ""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
# Standard ending — derives skipped from SHARE_COUNT, so a run with rsync gated off reports
|
||||||
|
# PARTIAL instead of "all complete".
|
||||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
orchestrator_summary "DAILY MAINTENANCE" "$WINDOW_START" "Daily Maintenance"
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
exit $?
|
||||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
||||||
"Daily Maintenance" "warning"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
@@ -12,7 +12,8 @@
|
|||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.cache/vv/d/)
|
# 1. conf_sync.sh — refresh partner conf cache in RAM (/tmp/varaverk/conf/),
|
||||||
|
# both directions: pull theirs, push ours
|
||||||
# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
||||||
# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
||||||
# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
||||||
@@ -182,23 +183,27 @@ echo ""
|
|||||||
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Conf Pull ━━━
|
# ━━━ Conf Sync ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
echo "━━━ $ICON_GEAR Conf Sync ━━━"
|
||||||
|
|
||||||
|
# Full sync, not --pull-only. The push half was written as an event-driven fast path for the
|
||||||
|
# conf-save hook, but no such hook was ever built — so outside array start nothing pushed this
|
||||||
|
# host's conf to its partners at all, and a partner's copy of our conf stayed at whatever it was
|
||||||
|
# when we last rebooted. Pull alone kept our view of them fresh while their view of us decayed.
|
||||||
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
||||||
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
||||||
warn "conf_sync.sh not found — skipping partner conf refresh"
|
warn "conf_sync.sh not found — skipping partner conf refresh"
|
||||||
else
|
else
|
||||||
_conf_args=("--pull-only")
|
_conf_args=()
|
||||||
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
|
||||||
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
|
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
|
||||||
echo "Partner conf cache refreshed ✅"
|
echo "Partner conf cache refreshed ✅"
|
||||||
JOB_PASS+=("conf_sync.sh --pull-only")
|
JOB_PASS+=("conf_sync.sh")
|
||||||
else
|
else
|
||||||
warn "Partner conf pull failed — cache may be stale"
|
warn "Partner conf sync failed — cache may be stale"
|
||||||
JOB_FAIL+=("conf_sync.sh --pull-only")
|
JOB_FAIL+=("conf_sync.sh")
|
||||||
fi
|
fi
|
||||||
unset _conf_args
|
unset _conf_args
|
||||||
fi
|
fi
|
||||||
@@ -352,20 +357,8 @@ if [[ "$SHOW_FULL" == true ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
|
||||||
warn "DRY RUN — no changes made"
|
# anything skipped or failed expands to the full block on its own.
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||||
if [[ "$SHOW_FULL" == true ]]; then
|
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
|
||||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
exit $?
|
||||||
else
|
|
||||||
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
|
||||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
||||||
"Intermediate Sync" "warning"
|
|
||||||
fi
|
|
||||||
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -307,18 +307,8 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped
|
||||||
warn "DRY RUN — no changes made"
|
# step visible rather than absent.
|
||||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
JOB_COUNT="$STEP"
|
||||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance"
|
||||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
exit $?
|
||||||
"Monthly Maintenance" "normal"
|
|
||||||
else
|
|
||||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
|
||||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
|
||||||
"Monthly Maintenance" "warning"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -98,6 +98,9 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|||||||
|
|
||||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||||
|
|
||||||
|
# Timed from here so the standard summary can report a real duration; this report had none.
|
||||||
|
REPORT_START=$(date +%s)
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -204,10 +207,9 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
# Standard ending. The configured section list is the denominator, so a report that quietly
|
||||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
# stopped producing one of its sections reads as skipped rather than simply not appearing.
|
||||||
"Sunday Morning Coffee Report" "warning"
|
JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}"
|
||||||
fi
|
[[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} ))
|
||||||
|
orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report"
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
exit $?
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -139,6 +139,10 @@ detect_hosts
|
|||||||
|
|
||||||
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
||||||
# from a healthy run. Fail loudly instead of silently doing no work.
|
# from a healthy run. Fail loudly instead of silently doing no work.
|
||||||
|
# This orchestrator never timed itself, so its summary could not report a duration. Set before
|
||||||
|
# any work so the figure means the cycle, not the tail of it.
|
||||||
|
CYCLE_START=$(date +%s)
|
||||||
|
|
||||||
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
||||||
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
||||||
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
||||||
@@ -223,15 +227,9 @@ done
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
|
# Quiet by default — 7-min cadence. Anything failed or skipped expands on its own.
|
||||||
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
JOB_COUNT="${#TRANSCODE_MANAGEMENT_SCRIPTS[@]}"
|
||||||
else
|
orchestrator_summary "TRANSCODE CYCLE" "${CYCLE_START:-$(date +%s)}" "Transcode Management" quiet
|
||||||
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
|
|
||||||
if [[ "$DRY_RUN" != true ]]; then
|
|
||||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
|
||||||
"Transcode Management" "warning"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Exit ━━━
|
# ━━━ Exit ━━━
|
||||||
|
|||||||
@@ -241,21 +241,15 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# Per-script detail only when there is something to read; the standard block carries the rest.
|
||||||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||||
echo ""
|
|
||||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
|
||||||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
else
|
|
||||||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
# Quiet by default at a 15-min cadence. The configured script list is the denominator, so a
|
||||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
# watchdog that silently stopped running one of its checks shows up as skipped.
|
||||||
"Watchdog Orchestrator" "warning"
|
JOB_COUNT="${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||||
exit 1
|
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||||
fi
|
orchestrator_summary "WATCHDOG CYCLE" "$CYCLE_START" "Watchdog Orchestrator" "$_mode"
|
||||||
|
exit $?
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -402,18 +402,14 @@ WINDOW_END=$(date +%s)
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary ━━━
|
# ━━━ Summary ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# Per-unit detail first — the standard block that follows carries the verdict and the counts, not
|
||||||
|
# the names, and knowing WHICH share failed is the whole point of reading a log.
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
||||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
|
||||||
|
|
||||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
@@ -422,19 +418,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section
|
||||||
|
# can no longer read as success — this is the run that printed "all complete — 0 shares synced".
|
||||||
echo ""
|
orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance"
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
exit $?
|
||||||
warn "DRY RUN — no changes made"
|
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
|
||||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
||||||
else
|
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
|
||||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
||||||
"Weekly Maintenance" "warning"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
@@ -181,14 +181,36 @@ if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
|
|||||||
warn "DRY RUN — would remove HOST1 key from $MIRROR:/root/.ssh/authorized_keys"
|
warn "DRY RUN — would remove HOST1 key from $MIRROR:/root/.ssh/authorized_keys"
|
||||||
H1_DONE=true
|
H1_DONE=true
|
||||||
else
|
else
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
# `\|…|d`, not `|…|d`. sed only accepts a custom address delimiter when it is
|
||||||
|
# introduced by a backslash; the bare form is a syntax error — "unknown command: `|'".
|
||||||
|
# A delimiter other than / is still required, because the key blob is base64 and
|
||||||
|
# routinely contains /.
|
||||||
|
#
|
||||||
|
# The error went to 2>/dev/null and `echo ok` ran anyway, so this reported
|
||||||
|
# "key removed ✅" on every run while removing nothing, and a cancelled onboard left
|
||||||
|
# HOST1's key live on the mirror. Report on what the remote actually did instead.
|
||||||
|
_cancel_out=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
||||||
"sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
|
"sed -i \"\\|${KEY_BLOB}|d\" /root/.ssh/authorized_keys || { echo sed-failed; exit 1; }
|
||||||
sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
|
chmod 600 /root/.ssh/authorized_keys 2>/dev/null
|
||||||
echo ok" 2>/dev/null | grep -q ok && {
|
sed -i \"/^${MIRROR_ID}_PHASE/d; /^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
|
||||||
echo "HOST1 key removed from $MIRROR authorized_keys ✅"
|
grep -qF '${KEY_BLOB}' /root/.ssh/authorized_keys 2>/dev/null && echo still-present || echo ok" 2>/dev/null)
|
||||||
H1_DONE=true
|
case "$_cancel_out" in
|
||||||
} || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
|
*ok*)
|
||||||
|
echo "HOST1 key removed from $MIRROR authorized_keys ✅"
|
||||||
|
H1_DONE=true
|
||||||
|
;;
|
||||||
|
*still-present*)
|
||||||
|
warn "HOST1 key still present in $MIRROR authorized_keys — remove it there manually"
|
||||||
|
;;
|
||||||
|
*sed-failed*)
|
||||||
|
warn "Could not edit authorized_keys on $MIRROR — remove HOST1 key there manually"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
unset _cancel_out
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -228,7 +250,9 @@ if [[ "$DIRECTION" == "h2" || "$DIRECTION" == "both" ]]; then
|
|||||||
warn "DRY RUN — would remove $MIRROR_SHORT key from $AUTH_KEYS"
|
warn "DRY RUN — would remove $MIRROR_SHORT key from $AUTH_KEYS"
|
||||||
H2_DONE=true
|
H2_DONE=true
|
||||||
else
|
else
|
||||||
sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && {
|
# chmod after: sed -i rewrites via a temp file, which lands under the umask and can
|
||||||
|
# leave the file 0666. sshd StrictModes then silently refuses every key in it.
|
||||||
|
sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && chmod 600 "$AUTH_KEYS" && {
|
||||||
echo "$MIRROR key removed from HOST1 authorized_keys ✅"
|
echo "$MIRROR key removed from HOST1 authorized_keys ✅"
|
||||||
H2_DONE=true
|
H2_DONE=true
|
||||||
} || warn "Failed to remove $MIRROR key from HOST1 authorized_keys"
|
} || warn "Failed to remove $MIRROR key from HOST1 authorized_keys"
|
||||||
|
|||||||
@@ -351,16 +351,53 @@ EOF
|
|||||||
) 200>"${file}.lock"
|
) 200>"${file}.lock"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Deliver the partnership state to the partner, at the path the partner actually reads, under
|
||||||
|
# both names its UI looks for.
|
||||||
|
#
|
||||||
|
# This used to be one scp to "root@ip:$local_file" — the LOCAL absolute path reused verbatim on
|
||||||
|
# the remote. That only works while both hosts install to the same place. HOST1 is on flash at
|
||||||
|
# /boot/config/plugins/varaverk and HOST2 is in appdata mode at /mnt/user/appdata/Varaverk, so
|
||||||
|
# the copy went to a directory HOST2 does not read and, more often, does not have — and the
|
||||||
|
# failure surfaced as "will propagate on next sync", which nothing does.
|
||||||
|
#
|
||||||
|
# Both names, because the page resolves one file per node: partnership_<that node's hostname>.db.
|
||||||
|
# The mirror needs partnership_<mirror>.db for its own card and partnership_<owner>.db for the
|
||||||
|
# owner's. Nothing on the mirror writes the first one during an owner-driven onboard, which is
|
||||||
|
# why a mirror that was fully onboarded still rendered as having no partnership at all.
|
||||||
|
# The content is symmetric — state/owner/mirror/onboarded — so one file serves as both.
|
||||||
push_state_to_remote() {
|
push_state_to_remote() {
|
||||||
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would push state file to remote"
|
warn "DRY RUN — would push state file to remote"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes \
|
|
||||||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
# Resolve where the partner keeps its state, from the partner. varaverk.cfg names its
|
||||||
echo "State file pushed to remote ✅" || \
|
# SCRIPTS_DIR; absent it, the flash default is the right guess for a stock install.
|
||||||
warn "Could not push state file to remote — will propagate on next sync"
|
local remote_sd remote_state_dir
|
||||||
|
remote_sd=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \
|
||||||
|
'grep -oP "(?<=SCRIPTS_DIR=\")[^\"]+" /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null' \
|
||||||
|
2>/dev/null | tr -d '\r')
|
||||||
|
[[ -z "$remote_sd" ]] && remote_sd="/boot/config/plugins/varaverk"
|
||||||
|
remote_state_dir="${remote_sd}/data/state"
|
||||||
|
|
||||||
|
local rc=0
|
||||||
|
local name
|
||||||
|
for name in "partnership_${REMOTE_SERVER_NAME}.db" "partnership_${LOCAL_SERVER_NAME}.db"; do
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$remote_ip" \
|
||||||
|
"mkdir -p '$remote_state_dir' && cat > '${remote_state_dir}/${name}'" \
|
||||||
|
< "$local_file" 2>/dev/null || rc=1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ $rc -eq 0 ]]; then
|
||||||
|
echo "State file pushed to $REMOTE_SERVER_NAME:${remote_state_dir} ✅"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
warn "Could not push state to $REMOTE_SERVER_NAME:${remote_state_dir} — it will keep showing"
|
||||||
|
warn " no partnership until this succeeds. Nothing retries this on a schedule."
|
||||||
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
read_remote_state() {
|
read_remote_state() {
|
||||||
@@ -467,6 +504,7 @@ do_ssh_key_revocation() {
|
|||||||
"grep -v '${our_comment}' /root/.ssh/authorized_keys \
|
"grep -v '${our_comment}' /root/.ssh/authorized_keys \
|
||||||
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
|
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
|
||||||
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
|
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
|
||||||
|
&& chmod 600 /root/.ssh/authorized_keys \
|
||||||
&& echo removed" 2>/dev/null | grep -q removed; then
|
&& echo removed" 2>/dev/null | grep -q removed; then
|
||||||
echo "Our pubkey revoked from $REMOTE_SERVER_NAME ✅"
|
echo "Our pubkey revoked from $REMOTE_SERVER_NAME ✅"
|
||||||
SSH_REVOKE_REMOTE_OK=true
|
SSH_REVOKE_REMOTE_OK=true
|
||||||
@@ -490,9 +528,16 @@ do_ssh_key_revocation() {
|
|||||||
SSH_REVOKE_LOCAL_OK=true
|
SSH_REVOKE_LOCAL_OK=true
|
||||||
elif [[ -f /root/.ssh/authorized_keys ]]; then
|
elif [[ -f /root/.ssh/authorized_keys ]]; then
|
||||||
if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then
|
if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then
|
||||||
|
# chmod after the mv, every time. `>` creates the temp file under the shell's umask
|
||||||
|
# and `mv` keeps the NEW file's mode, so this rewrite left authorized_keys 0666 on a
|
||||||
|
# filesystem that permits it. sshd's StrictModes then refuses every key in it without
|
||||||
|
# saying so to the client — the key is present, byte-correct, and inert, and the next
|
||||||
|
# onboard's SSH step fails with nothing in any Varaverk log to explain it. Only
|
||||||
|
# /var/log/syslog knows: "Authentication refused: bad ownership or modes".
|
||||||
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
|
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
|
||||||
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
|
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
|
||||||
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then
|
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys && \
|
||||||
|
chmod 600 /root/.ssh/authorized_keys; then
|
||||||
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
|
echo "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
|
||||||
SSH_REVOKE_LOCAL_OK=true
|
SSH_REVOKE_LOCAL_OK=true
|
||||||
else
|
else
|
||||||
@@ -532,6 +577,10 @@ gather_partner_fallback_containers() {
|
|||||||
|
|
||||||
# Start this server's own parked containers after partnership ends.
|
# Start this server's own parked containers after partnership ends.
|
||||||
start_own_stack() {
|
start_own_stack() {
|
||||||
|
# Returns non-zero if any container failed. It used to return whatever the loop's last
|
||||||
|
# docker start happened to produce, so a caller checking it learned nothing — and the
|
||||||
|
# offboard summary just printed "Step 6 — Own stack: started" either way.
|
||||||
|
local _rc=0
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_START Restart Own Stack ━━━"
|
echo "━━━ $ICON_START Restart Own Stack ━━━"
|
||||||
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
|
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
|
||||||
@@ -548,14 +597,20 @@ start_own_stack() {
|
|||||||
echo "$container started ✅"
|
echo "$container started ✅"
|
||||||
else
|
else
|
||||||
warn "$container failed to start — check manually"
|
warn "$container failed to start — check manually"
|
||||||
|
_rc=1
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
return "$_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Remove partnership containers on this server + their appdata bind-mount paths.
|
# Remove partnership containers on this server + their appdata bind-mount paths.
|
||||||
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
|
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
|
||||||
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
|
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
|
||||||
cleanup_partner_containers() {
|
cleanup_partner_containers() {
|
||||||
|
# Returns non-zero if any container or appdata path could not be removed. Previously the
|
||||||
|
# exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅"
|
||||||
|
# was printed over a container that failed to remove.
|
||||||
|
local _rc=0
|
||||||
declare -a containers=()
|
declare -a containers=()
|
||||||
gather_partner_fallback_containers containers
|
gather_partner_fallback_containers containers
|
||||||
|
|
||||||
@@ -586,8 +641,12 @@ cleanup_partner_containers() {
|
|||||||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||||||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
|
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
|
||||||
_PM_TRAP_STOPPED+=("$container")
|
_PM_TRAP_STOPPED+=("$container")
|
||||||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
|
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
|
||||||
echo "$container removed ✅" || warn "$container rm failed"
|
echo "$container removed ✅"
|
||||||
|
else
|
||||||
|
warn "$container rm failed"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
log "$container not found — skipping"
|
log "$container not found — skipping"
|
||||||
fi
|
fi
|
||||||
@@ -600,8 +659,14 @@ cleanup_partner_containers() {
|
|||||||
warn " DRY RUN — would rm -rf $path"
|
warn " DRY RUN — would rm -rf $path"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
if rm -rf "$path"; then
|
||||||
|
echo " Appdata removed: $path ✅"
|
||||||
|
else
|
||||||
|
warn " Failed to remove: $path"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
done <<< "$all_appdata_paths"
|
done <<< "$all_appdata_paths"
|
||||||
|
return "$_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
|
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
|
||||||
@@ -609,6 +674,10 @@ cleanup_partner_containers() {
|
|||||||
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
|
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
|
||||||
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
|
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
|
||||||
cleanup_owner_containers_on_mirror() {
|
cleanup_owner_containers_on_mirror() {
|
||||||
|
# Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly
|
||||||
|
# rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the
|
||||||
|
# containers on it are gone.
|
||||||
|
local _rc=0
|
||||||
local mirror_ip="$1"
|
local mirror_ip="$1"
|
||||||
local owner_short
|
local owner_short
|
||||||
owner_short=$(derive_short_name "$OWNER")
|
owner_short=$(derive_short_name "$OWNER")
|
||||||
@@ -639,24 +708,31 @@ cleanup_owner_containers_on_mirror() {
|
|||||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
||||||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||||||
|
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"docker stop '$container' >/dev/null 2>&1
|
"docker stop '$container' >/dev/null 2>&1
|
||||||
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||||||
grep -q removed && \
|
grep -q removed; then
|
||||||
echo "$container removed from $MIRROR ✅" || \
|
echo "$container removed from $MIRROR ✅"
|
||||||
|
else
|
||||||
warn "Failed to remove $container from $MIRROR"
|
warn "Failed to remove $container from $MIRROR"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
|
|
||||||
# Delete appdata on remote after container removal
|
# Delete appdata on remote after container removal
|
||||||
while IFS= read -r path; do
|
while IFS= read -r path; do
|
||||||
[[ -z "$path" ]] && continue
|
[[ -z "$path" ]] && continue
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
|
||||||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
echo " Appdata removed on $MIRROR: $path ✅"
|
||||||
|
else
|
||||||
warn " Failed to remove appdata on $MIRROR: $path"
|
warn " Failed to remove appdata on $MIRROR: $path"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
done <<< "$appdata_paths"
|
done <<< "$appdata_paths"
|
||||||
done <<< "$container_list"
|
done <<< "$container_list"
|
||||||
|
return "$_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
# SSH to mirror — start mirror's own parked containers.
|
# SSH to mirror — start mirror's own parked containers.
|
||||||
@@ -678,18 +754,23 @@ start_mirror_own_stack() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
|
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
|
||||||
|
local _rc=0
|
||||||
for container in "${mirror_own[@]}"; do
|
for container in "${mirror_own[@]}"; do
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would start $container on $MIRROR"
|
warn "DRY RUN — would start $container on $MIRROR"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||||||
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
||||||
grep -q started && \
|
grep -q started; then
|
||||||
echo "$container started on $MIRROR ✅" || \
|
echo "$container started on $MIRROR ✅"
|
||||||
|
else
|
||||||
warn "$container failed to start on $MIRROR — check manually"
|
warn "$container failed to start on $MIRROR — check manually"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
|
return "$_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Create the mirror's Emby admin account on the owner's deployed Emby.
|
# Create the mirror's Emby admin account on the owner's deployed Emby.
|
||||||
@@ -866,28 +947,70 @@ check_both_healthy() {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Returns non-zero when the mirror did NOT leave with current state, so the caller can say so.
|
||||||
|
# The whole point of this step is the guarantee in the summary — "mirror leaves with current
|
||||||
|
# Critical-Data" — and it used to print that unconditionally at the end of the function. An
|
||||||
|
# offboard with RSYNC_ENABLED=false logged "rsync globally disabled, skipping all syncs"
|
||||||
|
# immediately followed by "Final sync complete — mirror has current state ✅", and the summary
|
||||||
|
# scored Step 2 as a pass. The mirror left with whatever it happened to have.
|
||||||
do_final_sync() {
|
do_final_sync() {
|
||||||
log "Running final critical sync..."
|
log "Running final critical sync..."
|
||||||
|
local _synced=0 _failed=0
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
# Tier 1 only, deliberately. rsync.sh honours RSYNC_ENABLED and nothing else — the Tier 2
|
||||||
|
# gates belong to the orchestrators, so a direct call like this one runs whether or not
|
||||||
|
# CRITICAL_RSYNC_ENABLED is set. Checking Tier 2 here would refuse a final sync that works
|
||||||
|
# perfectly well, and the offboard's whole reason for existing is to get current auth data
|
||||||
|
# onto the mirror before the keys go.
|
||||||
|
#
|
||||||
|
# What was wrong was the accounting below it: a share was counted synced whenever Tier 1
|
||||||
|
# happened to be on, with rsync.sh's exit status ignored entirely. A transfer that failed
|
||||||
|
# on a full disk, an offline share or a refused connection still reported "Final sync ✅
|
||||||
|
# — mirror has current state", which is the one claim in this script somebody acts on.
|
||||||
|
local _gate_ok=true
|
||||||
|
if [[ "${RSYNC_ENABLED:-true}" != true ]]; then
|
||||||
|
warn "RSYNC_ENABLED=false — Tier 1 stops every rsync, no share can be sent"
|
||||||
|
_gate_ok=false
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$_gate_ok" != true ]]; then
|
||||||
|
_failed=1
|
||||||
|
elif [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||||||
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||||||
[[ -z "$_share" ]] && continue
|
[[ -z "$_share" ]] && continue
|
||||||
local _path="${_share%%|*}"
|
local _path="${_share%%|*}"
|
||||||
local _profile="${_share##*|}"
|
local _profile="${_share##*|}"
|
||||||
|
# Count on what rsync.sh returned, now that a closed gate can no longer reach here.
|
||||||
if [[ "$_path" == "$_profile" ]]; then
|
if [[ "$_path" == "$_profile" ]]; then
|
||||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
|
||||||
else
|
else
|
||||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
||||||
--profile="$_profile" --log
|
--profile="$_profile" --log
|
||||||
fi
|
fi
|
||||||
|
if [[ $? -eq 0 ]]; then (( _synced++ )) || true; else (( _failed++ )) || true; fi
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
|
||||||
|
_failed=1
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"
|
# ${#ARR[@]} and :- cannot be combined — bash rejects the whole word as "bad substitution",
|
||||||
|
# so this line aborted Step 2 of every offboard --dry-run with a shell error instead of
|
||||||
|
# printing. A count of an unset array is already 0, which is the only default needed.
|
||||||
|
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]} shares)"
|
||||||
|
return 0
|
||||||
fi
|
fi
|
||||||
warn "Final sync complete — mirror has current state ✅"
|
|
||||||
|
if [[ "$_failed" -gt 0 ]]; then
|
||||||
|
warn "Final sync did NOT complete — $MIRROR leaves with whatever state it already had"
|
||||||
|
[[ "${RSYNC_ENABLED:-true}" != true ]] && \
|
||||||
|
warn " RSYNC_ENABLED=false — the Tier 1 gate stopped it before any share was sent"
|
||||||
|
[[ "${CRITICAL_RSYNC_ENABLED:-true}" != true ]] && \
|
||||||
|
warn " CRITICAL_RSYNC_ENABLED=false — the Tier 2 gate stopped it"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "Final sync complete — $_synced share(s) sent, mirror has current state ✅"
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
# Safe master.conf modification with error handling — appends the key if not already present,
|
# Safe master.conf modification with error handling — appends the key if not already present,
|
||||||
|
|||||||
@@ -13,28 +13,32 @@
|
|||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
# OWNER PATH (10 steps)
|
# OWNER PATH (11 steps)
|
||||||
# Step 1: Stop rsync — halt any running sync before state changes
|
# Step 1: Stop rsync — halt any running sync before state changes
|
||||||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||||||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||||||
# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
# Step 4: Disarm sync gates — RSYNC/CRITICAL_RSYNC/CONF_SYNC/ARR_SYNC=false in master.conf,
|
||||||
|
# the exact inverse of onboard Step 9c
|
||||||
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
||||||
# Step 6: Restart own stack — bring up owner's own parked containers
|
# Step 6: Restart own stack — bring up owner's own parked containers
|
||||||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||||||
# Step 9: Revocation — Emby admin, SSH keys
|
# Step 9: Emby revocation — remove the mirror's Emby admin while Emby is still reachable
|
||||||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||||
# Tailscale — grace window then device removal (after state written)
|
# Tailscale — grace deadline recorded, device removed after it expires
|
||||||
|
# Step 11: SSH revocation — keys, both directions. Genuinely last: it is the step that
|
||||||
|
# removes the access every step above depends on
|
||||||
#
|
#
|
||||||
# MIRROR PATH (8 steps)
|
# MIRROR PATH (8 steps)
|
||||||
# Step 1: Stop rsync — halt any running sync
|
# Step 1: Stop rsync — halt any running sync
|
||||||
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
||||||
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
||||||
# Step 4: Fallback cleanup — remove fallback coverage containers
|
# Step 4: Fallback cleanup — remove fallback coverage containers
|
||||||
# Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
# Step 5: Disarm sync gates — same four gates as the owner path
|
||||||
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
||||||
# Step 7: Restart own stack — bring up own parked containers
|
# Step 7: Restart own stack — bring up own parked containers
|
||||||
# Step 8: SSH revocation — revoke keys both directions, write state, signal owner
|
# Step 8: Finalise — write INACTIVE, clear phase flags, signal owner, THEN revoke
|
||||||
|
# keys. Revocation is last because the signal needs the key
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
@@ -102,9 +106,12 @@
|
|||||||
# The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on —
|
# The mirror is added to the partnership blocklist, which rsync.sh checks and refuses on —
|
||||||
# stale access cannot survive the offboard.
|
# stale access cannot survive the offboard.
|
||||||
#
|
#
|
||||||
# Tailscale Grace Window
|
# Tailscale Grace Window Is a Deadline, Not a Sleep
|
||||||
# Device removal happens after state is written, not before, so the final state push
|
# Device removal happens after state is written, not before, so the final state push
|
||||||
# cannot be cut off by removing its own transport.
|
# cannot be cut off by removing its own transport. The grace period itself is recorded to
|
||||||
|
# STATE_DIR/tailscale_removal_due.db and the offboard returns. It used to sleep
|
||||||
|
# PARTNERSHIP_GRACE_HOURS inline — six hours by default — holding the lock and its job record
|
||||||
|
# open the whole time, reporting "running", and blocking any re-onboard behind it.
|
||||||
#
|
#
|
||||||
# Dry Run Support
|
# Dry Run Support
|
||||||
# --dry-run walks the full sequence reporting each step without executing any.
|
# --dry-run walks the full sequence reporting each step without executing any.
|
||||||
@@ -154,6 +161,19 @@ source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
|
|||||||
|
|
||||||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||||||
REASON="manual"
|
REASON="manual"
|
||||||
|
STEP_DISABLE_RSYNC_OK=true # both paths report it; only the mirror path re-initialised it
|
||||||
|
TAILSCALE_REMOVED=false # set only when remove_tailscale_device actually succeeds
|
||||||
|
|
||||||
|
# Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from
|
||||||
|
# MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded.
|
||||||
|
# An offboard that failed to remove a single container still reported a clean teardown.
|
||||||
|
STEP_LOCAL_CLEANUP_OK=true
|
||||||
|
STEP_OWN_STACK_OK=true
|
||||||
|
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
|
||||||
|
STEP_MIRROR_STACK_OK=true # or "skipped"
|
||||||
|
STEP_STATE_WRITE_OK=true
|
||||||
|
STEP_STATE_PUSH_OK=false # INACTIVE actually delivered to the mirror, or "skipped"
|
||||||
|
STEP_SETUP_PUSH_OK=false # cleared phase flags delivered to the mirror, or "skipped"
|
||||||
FILTERED_ARGS=()
|
FILTERED_ARGS=()
|
||||||
|
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
@@ -214,6 +234,57 @@ echo " Reason: $REASON"
|
|||||||
echo ""
|
echo ""
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── HELPER: disarm every sync gate a completed onboard armed ─────────────────────────────────
|
||||||
|
#
|
||||||
|
# The exact inverse of partnership_onboard.sh Step 9c, which sets RSYNC_ENABLED,
|
||||||
|
# CONF_SYNC_ENABLED and ARR_SYNC_ENABLED to true on a completed onboard. Offboard used to turn
|
||||||
|
# off CRITICAL_RSYNC_ENABLED and nothing else, which left the far more consequential Tier 1
|
||||||
|
# RSYNC_ENABLED switched on with no partner to sync to.
|
||||||
|
#
|
||||||
|
# That asymmetry read as safe and was not. Turning off one Tier 2 gate looks like "sync is off"
|
||||||
|
# in the summary, while INTERMEDIATE, DAILY, WEEKLY and FALLBACK rsync all stayed live behind an
|
||||||
|
# open Tier 1 — and CONF_SYNC_ENABLED kept a 4-hourly job reaching for a partner that had just
|
||||||
|
# been removed, failing and notifying each time.
|
||||||
|
#
|
||||||
|
# CRITICAL_RSYNC_ENABLED stays in the list. It is a Tier 2 gate and closing Tier 1 already stops
|
||||||
|
# it, but leaving it true would misreport the state to anyone reading the conf rather than the
|
||||||
|
# tier logic.
|
||||||
|
#
|
||||||
|
# FALLBACK_ENABLED is deliberately NOT here. Onboard does not arm it, so offboard has no business
|
||||||
|
# disarming it — it is the operator's switch, and the summary says so rather than moving it.
|
||||||
|
# ==============================================================================================
|
||||||
|
_VV_SYNC_GATES=(RSYNC_ENABLED CRITICAL_RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED)
|
||||||
|
|
||||||
|
# ── Stop any running rsync, and say honestly whether it worked ────────────────────────────────
|
||||||
|
#
|
||||||
|
# Both offboard paths called "$SCRIPTS_ROOT/Rsync/rsync_stop.sh". That file has never existed —
|
||||||
|
# rsync_stop.sh lives in System_Essentials/. With stderr sent to /dev/null the "No such file"
|
||||||
|
# went unseen, and the line below it printed "Rsync stopped ✅" unconditionally, so every
|
||||||
|
# offboard ever run reported stopping an rsync it had not touched. On the owner path the failed
|
||||||
|
# exit also set STEP_STOP_OK=false, which is why the summary said ❌ two lines under a ✅.
|
||||||
|
#
|
||||||
|
# One helper, one path, and the outcome is the return value.
|
||||||
|
stop_rsync_now() {
|
||||||
|
local script="$SCRIPTS_ROOT/System_Essentials/rsync_stop.sh"
|
||||||
|
if [[ ! -f "$script" ]]; then
|
||||||
|
warn "rsync_stop.sh not found at $script"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
bash "$script" --rsync-only
|
||||||
|
}
|
||||||
|
|
||||||
|
_disarm_sync_gates() {
|
||||||
|
local gate rc=0 conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||||||
|
for gate in "${_VV_SYNC_GATES[@]}"; do
|
||||||
|
# set_conf_bool, not update_master_conf: the latter rewrites the whole line and would
|
||||||
|
# strip the trailing comment that explains what each tier gates. Same helper onboard
|
||||||
|
# arms with, so arming and disarming are one operation in two directions.
|
||||||
|
set_conf_bool "$gate" "false" "$conf" || rc=1
|
||||||
|
done
|
||||||
|
return "$rc"
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
||||||
#
|
#
|
||||||
@@ -305,8 +376,12 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
if stop_rsync_now; then
|
||||||
echo "Rsync stopped ✅"
|
echo "Rsync stopped ✅"
|
||||||
|
else
|
||||||
|
warn "Could not stop rsync — a transfer may still be running into $OWNER"
|
||||||
|
STEP_STOP_RSYNC_OK=false
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would stop rsync"
|
warn "DRY RUN — would stop rsync"
|
||||||
fi
|
fi
|
||||||
@@ -335,16 +410,14 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
|
|
||||||
cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false
|
cleanup_partner_containers || STEP_FALLBACK_CLEANUP_OK=false
|
||||||
|
|
||||||
# ── Step 5: Disable critical sync ─────────────────────────────────────────────────────────
|
# ── Step 5: Disarm the sync gates ─────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━"
|
echo "━━━ $ICON_GEAR Step 5/8 — Disarm Sync Gates ━━━"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \
|
_disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false
|
||||||
warn "CRITICAL_RSYNC_ENABLED=false ✅" || \
|
|
||||||
{ warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; }
|
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
||||||
@@ -357,19 +430,31 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
||||||
|
|
||||||
do_ssh_key_revocation "${OWNER_IP:-}"
|
# State first, keys last — same ordering the owner path needed. Revocation used to run here,
|
||||||
|
# before the push below, so the mirror destroyed the key and then tried to tell the owner it
|
||||||
|
# had left using that key. The owner never heard, and the notify promised it would "finalise
|
||||||
|
# on next check" — a check that now had no way in.
|
||||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
write_state_file "$LOCAL_STATE_FILE" \
|
write_state_file "$LOCAL_STATE_FILE" \
|
||||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||||
echo "Local state: INACTIVE ✅"
|
echo "Local state: INACTIVE ✅"
|
||||||
add_to_blocklist "$OWNER" "$REASON"
|
add_to_blocklist "$OWNER" "$REASON"
|
||||||
|
|
||||||
|
# Inverse of onboard's write_onboard_phase. MIRROR_ID is this host on this path, and the
|
||||||
|
# flags are named for the mirror on both sides, so the same names clear here.
|
||||||
|
_setup_db="$(platform_setup_db_path)"
|
||||||
|
for _flag in "${MIRROR_ID}_PHASE1_DONE" "${MIRROR_ID}_PHASE2_DONE" "${MIRROR_ID}_KEY_READY"; do
|
||||||
|
clear_state_var "$_setup_db" "$_flag"
|
||||||
|
done
|
||||||
|
echo "Onboard phase flags cleared ✅"
|
||||||
|
unset _setup_db _flag
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would write INACTIVE state and blocklist $OWNER"
|
warn "DRY RUN — would write INACTIVE state, clear phase flags and blocklist $OWNER"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||||||
|
platform_push_setup_state 2>/dev/null || warn "Could not push cleared setup state to $OWNER"
|
||||||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
||||||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||||||
"Partnership" "normal"
|
"Partnership" "normal"
|
||||||
@@ -377,6 +462,9 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Last, for the reason above: everything before it needs the key.
|
||||||
|
do_ssh_key_revocation "${OWNER_IP:-}"
|
||||||
|
|
||||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
echo ""
|
echo ""
|
||||||
@@ -402,7 +490,7 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
||||||
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
||||||
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
||||||
echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")"
|
echo " Step 5 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
|
||||||
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
||||||
echo " Step 7 — Own stack: started"
|
echo " Step 7 — Own stack: started"
|
||||||
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
||||||
@@ -441,8 +529,14 @@ echo ""
|
|||||||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
|
if stop_rsync_now; then
|
||||||
echo "Rsync stopped ✅"
|
echo "Rsync stopped ✅"
|
||||||
|
else
|
||||||
|
warn "Could not stop rsync — a transfer may still be running into $MIRROR"
|
||||||
|
warn " Everything below changes state while data is still moving, which is the one"
|
||||||
|
warn " ordering this step exists to prevent. Check: ps -ef | grep rsync"
|
||||||
|
STEP_STOP_OK=false
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would stop rsync"
|
warn "DRY RUN — would stop rsync"
|
||||||
fi
|
fi
|
||||||
@@ -475,25 +569,24 @@ else
|
|||||||
(( WEBUI_FAILURES++ ))
|
(( WEBUI_FAILURES++ ))
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Step 4: Disable critical sync ─────────────────────────────────────────────────────────────
|
# ── Step 4: Disarm the sync gates ─────────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━"
|
echo "━━━ $ICON_GEAR Step 4/10 — Disarm Sync Gates ━━━"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
_disarm_sync_gates || STEP_DISABLE_RSYNC_OK=false
|
||||||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
warn "DRY RUN — would disarm ${_VV_SYNC_GATES[*]}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
||||||
|
|
||||||
cleanup_partner_containers
|
cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false
|
||||||
|
|
||||||
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
||||||
start_own_stack
|
start_own_stack || STEP_OWN_STACK_OK=false
|
||||||
|
|
||||||
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
@@ -501,64 +594,160 @@ echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━
|
|||||||
|
|
||||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false
|
||||||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false
|
||||||
else
|
else
|
||||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||||
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
||||||
|
STEP_REMOTE_CLEANUP_OK=skipped
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
||||||
|
|
||||||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||||
|
start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false
|
||||||
|
else
|
||||||
|
STEP_MIRROR_STACK_OK=skipped
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
# ── Step 9: Emby revocation ───────────────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
echo "━━━ $ICON_SHIELD Step 9/11 — Emby Revocation ━━━"
|
||||||
|
|
||||||
# Emby admin — before SSH key revocation while Emby still reachable
|
# Before SSH key revocation, while Emby is still reachable
|
||||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||||
|
|
||||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
|
||||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
|
||||||
|
|
||||||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||||||
# State is written after container cleanup and SSH revocation so that:
|
# Ahead of SSH revocation, which is now Step 11.
|
||||||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
#
|
||||||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
# "State Written Both Ends" is the guarantee this step exists for — neither side left believing
|
||||||
|
# the partnership is active. It could not deliver it: revocation used to run here in Step 9 and
|
||||||
|
# then this push authenticated with the key it had just destroyed, so every offboard ended
|
||||||
|
# "Could not push state file to remote — will propagate on next sync". There is no next sync;
|
||||||
|
# the gates are closed and the keys are gone. The mirror was left reading ACTIVE for ever.
|
||||||
|
#
|
||||||
|
# The original ordering note said state must follow revocation so a crash between steps 5–9
|
||||||
|
# re-runs from scratch rather than early-exiting on INACTIVE. Writing it one step later than the
|
||||||
|
# cleanup preserves that — the cleanup is still done before any state is recorded — while
|
||||||
|
# putting the push back inside the window where it can actually reach the mirror.
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
echo "━━━ $ICON_GEAR Step 10/11 — Write State ━━━"
|
||||||
|
|
||||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
write_state_file "$LOCAL_STATE_FILE" \
|
# Checked, because this is the record every other host and every later --check reads. A
|
||||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
# failed write here leaves both sides believing the partnership is still active while the
|
||||||
echo "Local state: INACTIVE ✅"
|
# summary says INACTIVE — the one line in the teardown that must not be assumed.
|
||||||
|
if write_state_file "$LOCAL_STATE_FILE" \
|
||||||
|
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then
|
||||||
|
echo "Local state: INACTIVE ✅"
|
||||||
|
else
|
||||||
|
error "Failed to write local state file — $MIRROR may still look ACTIVE here"
|
||||||
|
STEP_STATE_WRITE_OK=false
|
||||||
|
fi
|
||||||
add_to_blocklist "$MIRROR" "$REASON"
|
add_to_blocklist "$MIRROR" "$REASON"
|
||||||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
|
||||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
# The inverse of onboard's write_onboard_phase, which had none. Without this a completed
|
||||||
|
# offboard leaves the state file reading INACTIVE beside HOST*_PHASE1_DONE / _PHASE2_DONE
|
||||||
|
# still set — and the setup checklist and partnership card read the flags, not the state
|
||||||
|
# file, so a torn-down partnership went on presenting itself as fully onboarded.
|
||||||
|
#
|
||||||
|
# KEY_READY goes too: it means "a key is generated and waiting to be installed", which stops
|
||||||
|
# being true the moment Step 11 revokes both sides.
|
||||||
|
_setup_db="$(platform_setup_db_path)"
|
||||||
|
for _flag in "${MIRROR_ID}_PHASE1_DONE" "${MIRROR_ID}_PHASE2_DONE" "${MIRROR_ID}_KEY_READY"; do
|
||||||
|
clear_state_var "$_setup_db" "$_flag"
|
||||||
|
done
|
||||||
|
echo "Onboard phase flags cleared ✅"
|
||||||
|
unset _setup_db _flag
|
||||||
|
|
||||||
|
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||||
|
# Two separate pushes, and the summary used to report only the second. A run where the
|
||||||
|
# setup.db push failed and the state-file push succeeded printed "Could not push state
|
||||||
|
# file to remote" in the body and "Pushed to mirror: ✅" in the summary — describing
|
||||||
|
# different files with the same words. They are tracked apart now, because they fail
|
||||||
|
# apart: the mirror can be told the partnership is INACTIVE while keeping the phase
|
||||||
|
# flags that make its own UI still claim a finished onboard.
|
||||||
|
#
|
||||||
|
# Pushed after the flags are cleared, so the mirror receives the cleared file rather than
|
||||||
|
# the version that still claimed a finished onboard.
|
||||||
|
if platform_push_setup_state 2>/dev/null; then
|
||||||
|
STEP_SETUP_PUSH_OK=true
|
||||||
|
else
|
||||||
|
warn "Could not push cleared phase flags to $MIRROR — its wizard and partnership"
|
||||||
|
warn " card will keep showing a completed onboard until it is reinstalled or pulled"
|
||||||
|
fi
|
||||||
|
if push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||||
|
STEP_STATE_PUSH_OK=true
|
||||||
|
else
|
||||||
|
warn "Could not push INACTIVE to $MIRROR — it will keep reading ACTIVE until told otherwise"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
STEP_STATE_PUSH_OK=skipped
|
||||||
|
STEP_SETUP_PUSH_OK=skipped
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||||||
|
STEP_STATE_PUSH_OK=true
|
||||||
|
STEP_SETUP_PUSH_OK=true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Step 11: SSH key revocation ───────────────────────────────────────────────────────────────
|
||||||
|
# Genuinely last. Every step above needs working remote access — this is the one that takes it
|
||||||
|
# away, so it goes after the final thing that has to reach the mirror.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_SHIELD Step 11/11 — SSH Key Revocation ━━━"
|
||||||
|
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||||
|
|
||||||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
# The grace period is recorded as a deadline, not slept through.
|
||||||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
#
|
||||||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
# This used to `sleep $((PARTNERSHIP_GRACE_HOURS * 3600))` inline — six hours by default —
|
||||||
|
# holding the offboard's lock and its job record open the whole time, showing "running" to
|
||||||
|
# every status reader, and blocking any re-onboard behind the lock. Worse, the sleep ran
|
||||||
|
# even when removal was going to be a no-op: TAILSCALE_API_KEY and TAILSCALE_TAILNET are
|
||||||
|
# both empty here, so the six hours bought nothing at all.
|
||||||
|
#
|
||||||
|
# The offboard's own work is finished by this point. Writing the deadline lets the teardown
|
||||||
|
# complete now and leaves the removal to whoever reads the file — and makes the wait
|
||||||
|
# visible and cancellable instead of buried in a sleeping process.
|
||||||
|
_grace_h="${PARTNERSHIP_GRACE_HOURS:-6}"
|
||||||
|
if [[ "$_grace_h" -gt 0 ]] && [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||||
|
_due=$(( $(date +%s) + _grace_h * 3600 ))
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
printf 'host=%s\ndue=%s\ndue_human=%s\nreason=%s\n' \
|
||||||
sleep "$grace_seconds"
|
"$MIRROR" "$_due" "$(date -d "@$_due" '+%Y-%m-%d %H:%M:%S')" "$REASON" \
|
||||||
trap - SIGTERM SIGINT
|
> "${STATE_DIR}/tailscale_removal_due.db"
|
||||||
|
fi
|
||||||
|
warn "Grace period: $MIRROR stays on the tailnet until $(date -d "@$_due" '+%Y-%m-%d %H:%M') — recorded, not slept"
|
||||||
|
# No CLI entry point removes it yet, and there is deliberately no invented one here:
|
||||||
|
# remove_tailscale_device() is a partnership_manager.sh function with no --mode of its
|
||||||
|
# own, and it no-ops without credentials regardless. Say what is true.
|
||||||
|
if [[ -z "${TAILSCALE_API_KEY:-}" || -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||||||
|
warn "Automatic removal is not possible — TAILSCALE_API_KEY/TAILSCALE_TAILNET are unset; remove it in the Tailscale admin console"
|
||||||
|
else
|
||||||
|
warn "Removal after that is not yet automated — remove it in the Tailscale admin console"
|
||||||
|
fi
|
||||||
|
TAILSCALE_REMOVED=deferred
|
||||||
|
unset _grace_h _due
|
||||||
|
else
|
||||||
|
# No grace configured, or the mirror is already unreachable — remove now.
|
||||||
|
#
|
||||||
|
# Outcome recorded, not assumed. remove_tailscale_device returns 1 when TAILSCALE_API_KEY
|
||||||
|
# or TAILSCALE_TAILNET is unset — it warns "skipping Tailscale removal" and the summary
|
||||||
|
# went on to report "removed ✅" anyway, so an offboard that left the device on the
|
||||||
|
# tailnet said it had taken it off. Neither key is configured here, so that was every run.
|
||||||
|
if remove_tailscale_device "$MIRROR"; then
|
||||||
|
TAILSCALE_REMOVED=true
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
remove_tailscale_device "$MIRROR"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Backup handover notification
|
# Backup handover notification
|
||||||
@@ -597,24 +786,60 @@ _revoke_status() {
|
|||||||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
||||||
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
||||||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||||||
echo " Step 4 — Disable sync: ✅"
|
echo " Step 4 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
|
||||||
echo " Step 5 — Local cleanup: ✅"
|
echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")"
|
||||||
echo " Step 6 — Own stack: started"
|
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
|
||||||
echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
|
||||||
echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
|
||||||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
echo " Step 9 — Emby revoked: $(_ok "${STEP_EMBY_OK:-true}")"
|
||||||
echo " Step 10 — State: INACTIVE ✅"
|
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
|
||||||
|
echo " Step 10 — INACTIVE pushed: $( [[ "$STEP_STATE_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_STATE_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still reads ACTIVE"; } )"
|
||||||
|
echo " Step 10 — Phase flags pushed: $( [[ "$STEP_SETUP_PUSH_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_SETUP_PUSH_OK" == true ]] && echo "✅" || echo "⚠️ $MIRROR still shows a finished onboard"; } )"
|
||||||
|
echo " Step 11 — Keys revoked: $(_revoke_status)"
|
||||||
echo ""
|
echo ""
|
||||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-false}" == true ]]; then
|
||||||
echo " Tailscale: $MIRROR removed ✅"
|
case "$TAILSCALE_REMOVED" in
|
||||||
|
true) echo " Tailscale: $MIRROR removed ✅" ;;
|
||||||
|
deferred) echo " Tailscale: $MIRROR kept until the grace period expires — see ${STATE_DIR}/tailscale_removal_due.db" ;;
|
||||||
|
*) echo " Tailscale: $MIRROR NOT removed ⚠ — still on the tailnet (needs TAILSCALE_API_KEY + TAILSCALE_TAILNET)" ;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
# Named because it is the one partnership switch neither onboard nor offboard moves, so it
|
||||||
|
# survives an offboard still true and there is nothing else that would ever mention it.
|
||||||
|
[[ "${FALLBACK_ENABLED:-false}" == true ]] && \
|
||||||
|
echo " FALLBACK_ENABLED is still true — nothing left to fail over to, turn it off yourself"
|
||||||
echo ""
|
echo ""
|
||||||
echo " $MIRROR leaves with:"
|
echo " $MIRROR leaves with:"
|
||||||
echo " ✓ Current auth config (final sync)"
|
# Each line reads the step that produced it. "✓ Current auth config (final sync)" was a
|
||||||
echo " ✓ Auth WebUIs → localhost"
|
# literal, so an offboard whose Step 2 had just warned "Final sync did NOT complete" still
|
||||||
|
# closed by telling the operator the mirror held current auth data. That is the one claim here
|
||||||
|
# somebody might act on — it is the difference between a partner that can stand alone and one
|
||||||
|
# carrying a stale copy of the auth stack.
|
||||||
|
if [[ "$STEP_SYNC_OK" == true ]]; then
|
||||||
|
echo " ✓ Current auth config (final sync)"
|
||||||
|
else
|
||||||
|
echo " ✗ Auth config NOT synced — it keeps whatever it already had (see Step 2)"
|
||||||
|
fi
|
||||||
|
if [[ "${WEBUI_FAILURES:-0}" -eq 0 ]]; then
|
||||||
|
echo " ✓ Auth WebUIs → localhost"
|
||||||
|
else
|
||||||
|
echo " ✗ ${WEBUI_FAILURES} auth WebUI(s) still point at $LOCAL_SERVER_NAME — fix by hand there"
|
||||||
|
fi
|
||||||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||||||
echo ""
|
echo ""
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
|
||||||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
# The verdict is derived, never asserted. "DONE — clean separation complete ✅" printed
|
||||||
|
# unconditionally, under a summary that had already shown two failed steps.
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — no changes made"
|
||||||
|
elif [[ "$STEP_STOP_OK" == true && "$STEP_SYNC_OK" == true \
|
||||||
|
&& "$STEP_STATE_PUSH_OK" != false && "$STEP_SETUP_PUSH_OK" != false \
|
||||||
|
&& "${WEBUI_FAILURES:-0}" -eq 0 ]]; then
|
||||||
|
echo "$ICON_DONE DONE — clean separation complete ✅"
|
||||||
|
else
|
||||||
|
warn "$ICON_DONE Offboard finished with unresolved steps — the partnership is ended, but"
|
||||||
|
warn " the ❌ lines above did not happen. Re-read them before re-onboarding."
|
||||||
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -22,6 +22,12 @@
|
|||||||
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
# Step 1: SSH key setup — generate keypair, install on mirror, update conf
|
||||||
# Step 1b: Docker network — ensure varaverk docker network exists on mirror
|
# Step 1b: Docker network — ensure varaverk docker network exists on mirror
|
||||||
# Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent)
|
# Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent)
|
||||||
|
# Step 1c2:Permit mirror — drop the mirror from the partnership blocklist a previous
|
||||||
|
# offboard wrote, or Step 1e's rsync is refused
|
||||||
|
# Step 1d: Sync gates — Tier 1 + CONF_SYNC + ARR_SYNC on, every Tier 2 rsync gate
|
||||||
|
# off. Here, not at the end, because Step 1e is an rsync
|
||||||
|
# Step 1e: Auth appdata — rsync PARTNERSHIP_PROVISION_SHARES to the mirror BEFORE the
|
||||||
|
# containers that read it exist. The only rsync an onboard does
|
||||||
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
# Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing
|
||||||
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
# Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror
|
||||||
# Mariadb/Redis health-checked before Authelia deploys
|
# Mariadb/Redis health-checked before Authelia deploys
|
||||||
@@ -32,10 +38,19 @@
|
|||||||
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
# Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby
|
||||||
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
# Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh)
|
||||||
# Step 9b: Webhook setup — register download webhook in arrs on both servers
|
# Step 9b: Webhook setup — register download webhook in arrs on both servers
|
||||||
# Step 9c: Media seed — rsync all DAILY_SYNC_SHARES to mirror (--seed)
|
# Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
||||||
# prevents arrs treating every file as missing after bootstrap
|
|
||||||
# Step 9d: Webhook listener — start listener on mirror (runs continuously, no reboot needed)
|
|
||||||
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
# Step 10: Conf push — push master.conf + setup state to all listed hosts
|
||||||
|
# Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it
|
||||||
|
# would discover are actually deployed there
|
||||||
|
# Step 12: Container grouping — reproduce this host's folder layout on the mirror (Arrs
|
||||||
|
# Stack, Networking, Databases…) for the containers deployed
|
||||||
|
# there; only unfiled or PARTNERSHIP_FALLBACK_ONLY entries go
|
||||||
|
# to "<OwnerShort>-Fallback". Icon resolved here and passed
|
||||||
|
# over: the mirror has no Emby key
|
||||||
|
#
|
||||||
|
# Phase 3 (media seed) is NOT part of the above. Onboard ends at Phase 2 — connected, running,
|
||||||
|
# and saying so. Seeding is a separate operator-triggered phase; see --phase3-only. With
|
||||||
|
# MEDIA_SEED_ENABLED=false there is no Phase 3 and the model is two phases.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
@@ -150,16 +165,27 @@
|
|||||||
# Partnership/partnership_onboard.sh --skip-webhook-setup
|
# Partnership/partnership_onboard.sh --skip-webhook-setup
|
||||||
# Skip webhook registration in arrs (Step 9b)
|
# Skip webhook registration in arrs (Step 9b)
|
||||||
#
|
#
|
||||||
# Partnership/partnership_onboard.sh --skip-media-seed
|
# Partnership/partnership_onboard.sh --phase3-only
|
||||||
# Skip initial media share rsync to mirror (Step 9c)
|
# OWNER: media seed only. Refuses unless Phase 2 is done and MEDIA_SEED_ENABLED is true.
|
||||||
# Use when mirror already has files or you want to seed manually
|
# Sets the seeding gate posture (Tier 1 open, every Tier 2 gate closed) and dispatches
|
||||||
|
# Rsync/media_seed.sh detached. Returns in seconds; the seed runs for weeks.
|
||||||
|
#
|
||||||
|
# Partnership/partnership_onboard.sh --skip-appdata-provision
|
||||||
|
# Skip the pre-container auth appdata rsync (Step 1e)
|
||||||
|
# The mirror's auth containers then start against whatever is already there
|
||||||
#
|
#
|
||||||
# Partnership/partnership_onboard.sh --skip-webhook-listener
|
# Partnership/partnership_onboard.sh --skip-webhook-listener
|
||||||
# Skip starting webhook listener on mirror (Step 9d)
|
# Skip starting webhook listener on mirror (Step 9e)
|
||||||
# Listener will start automatically on next array restart
|
# Listener will start automatically on next array restart
|
||||||
#
|
#
|
||||||
|
# Partnership/partnership_onboard.sh --no-arm
|
||||||
|
# Leave every sync gate as it is (Step 1d). Step 1e then has no Tier 1 and cannot provision.
|
||||||
|
# Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose
|
||||||
|
# shares are not populated yet.
|
||||||
|
#
|
||||||
# Partnership/partnership_onboard.sh --phase1-only
|
# Partnership/partnership_onboard.sh --phase1-only
|
||||||
# OWNER only: SSH key exchange + conf push. Safe to run before HOST2 has Varaverk.
|
# OWNER only: SSH key exchange, conf push, docker network, partner conf cache.
|
||||||
|
# Safe to run before HOST2 has Varaverk — all of it needs docker and SSH, not the plugin.
|
||||||
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
# Writes HOST2_PHASE1_DONE=true to varaverk_setup.db.
|
||||||
#
|
#
|
||||||
# Partnership/partnership_onboard.sh --phase2-only
|
# Partnership/partnership_onboard.sh --phase2-only
|
||||||
@@ -184,10 +210,13 @@ SKIP_ARR_STACK=false
|
|||||||
SKIP_SERVICES_STACK=false
|
SKIP_SERVICES_STACK=false
|
||||||
SKIP_ARR_SYNC=false
|
SKIP_ARR_SYNC=false
|
||||||
SKIP_WEBHOOK_SETUP=false
|
SKIP_WEBHOOK_SETUP=false
|
||||||
SKIP_MEDIA_SEED=false
|
|
||||||
SKIP_WEBHOOK_LISTENER=false
|
SKIP_WEBHOOK_LISTENER=false
|
||||||
PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed)
|
PHASE1_ONLY=false # OWNER: SSH + conf push only (mirror not yet installed)
|
||||||
|
# MIRROR: SSH key install only, no owner notification
|
||||||
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards)
|
||||||
|
PHASE3_ONLY=false # OWNER: media seed only — a separate decision, see the PHASE 3 block
|
||||||
|
SKIP_ARM=false # leave the sync gates as they are — see Step 1d
|
||||||
|
SKIP_APPDATA_PROVISION=false # skip the pre-container auth appdata sync — see Step 1e
|
||||||
FILTERED_ARGS=()
|
FILTERED_ARGS=()
|
||||||
|
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
@@ -199,10 +228,12 @@ for arg in "$@"; do
|
|||||||
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
--skip-services-stack) SKIP_SERVICES_STACK=true ;;
|
||||||
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
--skip-arr-sync) SKIP_ARR_SYNC=true ;;
|
||||||
--skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;;
|
--skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;;
|
||||||
--skip-media-seed) SKIP_MEDIA_SEED=true ;;
|
--skip-appdata-provision) SKIP_APPDATA_PROVISION=true ;;
|
||||||
--skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;;
|
--skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;;
|
||||||
--phase1-only) PHASE1_ONLY=true ;;
|
--phase1-only) PHASE1_ONLY=true ;;
|
||||||
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
--phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;;
|
||||||
|
--phase3-only) PHASE3_ONLY=true; SKIP_SSH=true ;;
|
||||||
|
--no-arm) SKIP_ARM=true ;;
|
||||||
*) FILTERED_ARGS+=("$arg") ;;
|
*) FILTERED_ARGS+=("$arg") ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
@@ -238,9 +269,15 @@ write_onboard_phase() {
|
|||||||
local state_file="$(platform_setup_db_path)"
|
local state_file="$(platform_setup_db_path)"
|
||||||
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
[[ "$DRY_RUN" == true ]] && { warn "DRY RUN — would write ${key}=true"; return 0; }
|
||||||
set_state_var "$state_file" "$key" "true"
|
set_state_var "$state_file" "$key" "true"
|
||||||
platform_push_setup_state
|
# The push now reports failure, so it is warned about rather than returned. This function's
|
||||||
|
# contract is "the phase is recorded here" — the flag is on local disk either way, and an
|
||||||
|
# unreachable partner must not make a completed phase look like it did not happen.
|
||||||
|
platform_push_setup_state \
|
||||||
|
|| warn "Phase flag ${key} written locally but not delivered to the partner"
|
||||||
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_FALLBACK Partnership Onboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
echo ""
|
echo ""
|
||||||
@@ -297,14 +334,47 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
echo " Mirror sets up SSH keys, then notifies Owner to run Phase 2."
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# Tested before attempted, the same guard the owner's phase 1 has had all along.
|
||||||
|
#
|
||||||
|
# The key step is a TERMINAL step by design — ssh_setup.sh runs ssh-copy-id, which prompts for
|
||||||
|
# the owner's root password on a first install. Once the operator has done that in a terminal,
|
||||||
|
# pressing ▶ Onboard ran the whole thing again: another ssh-copy-id, this time from the WebGUI
|
||||||
|
# with no TTY to answer the prompt, which fails and aborts the run at Step 1 — so the button
|
||||||
|
# whose entire job is Step 2 could never reach it.
|
||||||
|
#
|
||||||
|
# Working SSH is the actual precondition, not "have we run the setup script". If it already
|
||||||
|
# works there is nothing to install, whichever route installed it.
|
||||||
|
OWNER_IP_PRE=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
|
||||||
if [[ "$SKIP_SSH" == true ]]; then
|
if [[ "$SKIP_SSH" == true ]]; then
|
||||||
warn "Skipping SSH setup (--skip-ssh)"
|
warn "Skipping SSH setup (--skip-ssh)"
|
||||||
|
elif [[ -n "$OWNER_IP_PRE" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$OWNER_IP_PRE" exit 0 2>/dev/null; then
|
||||||
|
echo "SSH to $OWNER already works ✅ — key already installed, skipping setup"
|
||||||
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
elif bash "$SCRIPT_DIR/ssh_setup.sh" "${EXTRA_FLAGS[@]}"; then
|
||||||
echo "SSH key ready ✅"
|
echo "SSH key ready ✅"
|
||||||
else
|
else
|
||||||
error "SSH key setup failed"
|
error "SSH key setup failed"
|
||||||
|
error "Install the key from a terminal on this host — ssh-copy-id needs $OWNER's password,"
|
||||||
|
error "and a WebGUI button has no way to answer that prompt:"
|
||||||
|
error " bash $SCRIPTS_ROOT/Partnership/partnership_onboard.sh --phase1-only"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
unset OWNER_IP_PRE
|
||||||
|
|
||||||
|
# Stop after the key when asked. ssh_setup.sh runs ssh-copy-id, which prompts for the
|
||||||
|
# owner's root password on a first install — answerable in a terminal, never from the
|
||||||
|
# WebGUI button, which is why the mirror's panel sends the operator to a terminal for
|
||||||
|
# exactly this step and nothing more. The flag was parsed but only ever honoured on the
|
||||||
|
# owner path, so a mirror asked for phase 1 silently ran the whole thing.
|
||||||
|
if [[ "$PHASE1_ONLY" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY MIRROR PHASE 1 COMPLETE ━━━━━"
|
||||||
|
echo " SSH key: ready"
|
||||||
|
echo " Next: press ▶ Onboard on the Partnership tab to notify $OWNER"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
echo "━━━ Step 2/2 — Notify Owner to Run Phase 2 ━━━"
|
||||||
@@ -318,17 +388,77 @@ if [[ "$AM_MIRROR" == true ]]; then
|
|||||||
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
|
OWNER_SCRIPTS_DIR=$(resolve_remote_scripts_dir "$OWNER_IP")
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
|
warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2 via run_job.sh"
|
||||||
PHASE2_TRIGGERED=true
|
|
||||||
elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
|
||||||
"nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
|
|
||||||
2>/dev/null | grep -q triggered; then
|
|
||||||
echo "Phase 2 triggered on $OWNER ✅"
|
|
||||||
log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
|
|
||||||
PHASE2_TRIGGERED=true
|
PHASE2_TRIGGERED=true
|
||||||
else
|
else
|
||||||
warn "Could not auto-trigger Phase 2 on $OWNER"
|
# Launched through run_job.sh, the same path cron and api/run.php use, so Phase 2
|
||||||
|
# gets a stat file, a job log and a Scheduler entry on the owner. It used to be a
|
||||||
|
# bare `nohup … > /tmp/vv_phase2_onboard.log`, which ran fine and left the owner's
|
||||||
|
# entire half of onboarding invisible to its own WebGUI — no job record, nothing
|
||||||
|
# under /var/log/varaverk, nothing for the operator to look at when asking why
|
||||||
|
# pressing Onboard here appeared to do nothing.
|
||||||
|
#
|
||||||
|
# setsid, not bare nohup: the job must lead its own process group so api/stop.php
|
||||||
|
# can signal the whole tree. api/run.php carries the same note for the same reason.
|
||||||
|
#
|
||||||
|
# Reported triggered only after the stat file proves run_job.sh actually started.
|
||||||
|
# The old `& echo triggered` printed unconditionally — it would have claimed success
|
||||||
|
# for a path that does not exist on the owner, which is exactly the failure mode a
|
||||||
|
# mirror in a different storage mode hits.
|
||||||
|
# Its own timeout, not SSH_TIMEOUT: the remote waits for the runner to prove itself,
|
||||||
|
# and 15s would cut that short and report a healthy launch as a failure.
|
||||||
|
_phase2_out=$(timeout 40 ssh -i "$SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
|
||||||
|
"bash -s -- $(printf '%q' "$OWNER_SCRIPTS_DIR")" 2>/dev/null <<'PHASE2_TRIGGER'
|
||||||
|
sd="$1"
|
||||||
|
runner="$sd/Plugin/unraid/run_job.sh"
|
||||||
|
script="$sd/Partnership/partnership_onboard.sh"
|
||||||
|
stat_file="/var/log/varaverk/Partnership/partnership_onboard.json"
|
||||||
|
|
||||||
|
[ -f "$runner" ] || { echo "missing-runner:$runner"; exit 1; }
|
||||||
|
[ -f "$script" ] || { echo "missing-script:$script"; exit 1; }
|
||||||
|
|
||||||
|
# Absolute, not "newer than the file we saw a moment ago". A previous run's stat file rewritten
|
||||||
|
# inside the same second would compare equal and read as a failed launch.
|
||||||
|
t0=$(date +%s)
|
||||||
|
|
||||||
|
setsid nohup bash "$runner" "Partnership/partnership_onboard.sh" "$script" \
|
||||||
|
--manual --phase2-only >/dev/null 2>&1 </dev/null &
|
||||||
|
|
||||||
|
# run_job.sh writes the stat file before it execs the script, so a stat file dated at or after
|
||||||
|
# the moment we launched is proof the runner is alive — rather than proof the ssh call returned.
|
||||||
|
for _ in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do
|
||||||
|
sleep 1
|
||||||
|
[ -f "$stat_file" ] || continue
|
||||||
|
now=$(date -r "$stat_file" +%s 2>/dev/null || echo 0)
|
||||||
|
[ "$now" -ge "$t0" ] && { echo triggered; exit 0; }
|
||||||
|
done
|
||||||
|
echo start-failed
|
||||||
|
exit 1
|
||||||
|
PHASE2_TRIGGER
|
||||||
|
)
|
||||||
|
# Failure patterns are matched first, and they echo a path back. An owner whose
|
||||||
|
# SCRIPTS_DIR happened to contain the word "triggered" would otherwise satisfy a
|
||||||
|
# leading *triggered* glob and report success for a launch that never happened.
|
||||||
|
case "$_phase2_out" in
|
||||||
|
missing-runner:*)
|
||||||
|
error "Phase 2 not started — no run_job.sh at ${_phase2_out#missing-runner:} on $OWNER"
|
||||||
|
;;
|
||||||
|
missing-script:*)
|
||||||
|
error "Phase 2 not started — no partnership_onboard.sh at ${_phase2_out#missing-script:} on $OWNER"
|
||||||
|
;;
|
||||||
|
start-failed)
|
||||||
|
error "Phase 2 launch on $OWNER did not produce a job record — check run_job.sh there"
|
||||||
|
;;
|
||||||
|
triggered)
|
||||||
|
echo "Phase 2 triggered on $OWNER ✅"
|
||||||
|
log "Watch on $OWNER: Scheduler tab, or tail -f /var/log/varaverk/Partnership/partnership_onboard.log"
|
||||||
|
PHASE2_TRIGGERED=true
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
warn "Could not auto-trigger Phase 2 on $OWNER"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
warn "Cannot resolve $OWNER Tailscale IP"
|
warn "Cannot resolve $OWNER Tailscale IP"
|
||||||
@@ -356,10 +486,129 @@ MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
|||||||
log "Mirror: $MIRROR ($MIRROR_IP)"
|
log "Mirror: $MIRROR ($MIRROR_IP)"
|
||||||
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
[[ "$PHASE1_ONLY" == true ]] && log "Mode: Phase 1 only (SSH + conf push)"
|
||||||
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
[[ "$PHASE2_ONLY" == true ]] && log "Mode: Phase 2 only (containers + arr + onboard)"
|
||||||
|
[[ "$PHASE3_ONLY" == true ]] && log "Mode: Phase 3 only (media seed)"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── PHASE 3: MEDIA SEED ───────────────────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# The partnership is finished at the end of Phase 2 — connected, running, and saying so. Phase 3
|
||||||
|
# is not part of it. It is the separate decision to fill the partner's disks by rsync, and it is
|
||||||
|
# operator-triggered because a first seed of a full library is weeks of transfer and tens of
|
||||||
|
# terabytes of somebody else's free space.
|
||||||
|
#
|
||||||
|
# The seed used to be a step inside Phase 2. Inline it held the onboard open for the entire
|
||||||
|
# transfer, which meant the phase-2 flag, the conf push, discovery and container grouping all
|
||||||
|
# waited on it, and both hosts described an unfinished onboard over a partnership that was
|
||||||
|
# already live. Detaching it fixed the blocking but left the seed starting on its own, which is
|
||||||
|
# still the wrong default: the machine decided to move 28 TB because an onboard succeeded.
|
||||||
|
#
|
||||||
|
# With MEDIA_SEED_ENABLED=false there is no Phase 3 at all and the model is two phases. That is
|
||||||
|
# the point of the toggle — not a seed that runs and does nothing, an absent phase.
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$PHASE3_ONLY" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SYNC PHASE 3 — MEDIA SEED — $MY_ID → $MIRROR ━━━━━"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||||||
|
_seed_gate=$(grep -m1 -E '^[[:space:]]*MEDIA_SEED_ENABLED=' "$_master_conf" 2>/dev/null \
|
||||||
|
| cut -d= -f2- | cut -d'#' -f1 | tr -d '"'"'" | tr -d '[:space:]')
|
||||||
|
|
||||||
|
if [[ -n "$_seed_gate" && "$_seed_gate" != "true" ]]; then
|
||||||
|
error "MEDIA_SEED_ENABLED is '$_seed_gate' — there is no Phase 3 on this host"
|
||||||
|
error " Onboarding is two phases while seeding is off. Arm the toggle in master.conf"
|
||||||
|
error " if you want the partner's library filled by rsync."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
_setup_db="$(platform_setup_db_path)"
|
||||||
|
if ! grep -q "^${MIRROR_ID}_PHASE2_DONE=true" "$_setup_db" 2>/dev/null; then
|
||||||
|
error "$MIRROR has not completed Phase 2 — seeding a partner that is not onboarded"
|
||||||
|
error " would push media at a host with no containers to serve it. Finish Phase 2 first."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
unset _setup_db
|
||||||
|
|
||||||
|
# Seeding posture, set before the transfer rather than after it. Tier 1 has to be open or
|
||||||
|
# rsync.sh refuses every share; every Tier 2 gate is closed so the scheduled orchestrators
|
||||||
|
# are not competing for the same link and the same disks for the next several weeks. This is
|
||||||
|
# also the posture the seed leaves behind — global on, per-orchestrator off — so finishing
|
||||||
|
# Phase 3 does not quietly hand the schedule back.
|
||||||
|
echo "━━━ Gate posture ━━━"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would set RSYNC_ENABLED true and every Tier 2 rsync gate false"
|
||||||
|
else
|
||||||
|
cp -a "$_master_conf" "${_master_conf}.bak-phase3-$(date +%Y%m%d-%H%M%S)"
|
||||||
|
set_conf_bool RSYNC_ENABLED "true" "$_master_conf" \
|
||||||
|
|| { error "Could not open Tier 1 — the seed would move nothing"; exit 1; }
|
||||||
|
for _gate in CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED \
|
||||||
|
WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED; do
|
||||||
|
set_conf_bool "$_gate" "false" "$_master_conf" \
|
||||||
|
|| warn "Could not close $_gate — a scheduled sync may run beside the seed"
|
||||||
|
done
|
||||||
|
unset _gate
|
||||||
|
echo " Tier 1 open, every Tier 2 gate closed ✅"
|
||||||
|
fi
|
||||||
|
unset _master_conf _seed_gate
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "━━━ Dispatch ━━━"
|
||||||
|
_seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh"
|
||||||
|
_runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would dispatch Rsync/media_seed.sh"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if [[ ! -f "$_seed_script" || ! -f "$_runner" ]]; then
|
||||||
|
error "Seed runner not found at $_seed_script"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# setsid, not nohup + &: the seed must outlive this script and must not share its process
|
||||||
|
# group, or stopping Phase 3 takes the transfer down with it.
|
||||||
|
_seed_flags=()
|
||||||
|
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
||||||
|
setsid bash "$_runner" "Rsync/media_seed.sh" "$_seed_script" "${_seed_flags[@]}" \
|
||||||
|
>/dev/null 2>&1 < /dev/null &
|
||||||
|
disown 2>/dev/null || true
|
||||||
|
|
||||||
|
# Report the record, not the launch. run_job.sh writes its stat file before running the
|
||||||
|
# script, so a fresh "running" record is the difference between dispatched and attempted.
|
||||||
|
_seed_stat="/var/log/varaverk/Rsync/media_seed.json"
|
||||||
|
_dispatched=false
|
||||||
|
for _i in $(seq 1 10); do
|
||||||
|
if [[ -f "$_seed_stat" ]] \
|
||||||
|
&& grep -q '"status":"running"' "$_seed_stat" 2>/dev/null \
|
||||||
|
&& (( $(date +%s) - $(stat -c %Y "$_seed_stat") < 60 )); then
|
||||||
|
_dispatched=true
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY PHASE 3 SUMMARY ━━━━━"
|
||||||
|
if [[ "$_dispatched" == true ]]; then
|
||||||
|
echo " Seed: running as job Rsync/media_seed.sh ✅"
|
||||||
|
echo " Shares: ${#DAILY_SYNC_SHARES[@]}"
|
||||||
|
echo " Gates: Tier 1 open · every Tier 2 closed"
|
||||||
|
echo ""
|
||||||
|
echo " A first full seed runs for days. Watch it on the Partnership tab; the"
|
||||||
|
echo " partnership itself has been live since Phase 2 and does not depend on this."
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
error " Seed did not start — check /var/log/varaverk/Rsync/media_seed.log"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
STEP_SSH_OK=false
|
STEP_SSH_OK=false
|
||||||
STEP_NETWORK_OK=false
|
STEP_NETWORK_OK=false
|
||||||
|
LOCAL_SETUP_OK=true # partnership_manager --local-only; the summary claimed done ✅ regardless
|
||||||
|
PHASE1_NET_OK=false # Phase 1 only — network created on the mirror before any deploy
|
||||||
|
PHASE1_CACHE_OK=false # Phase 1 only — our conf pushed into the mirror's RAM cache
|
||||||
STEP_STOP_AUTH_OK=true
|
STEP_STOP_AUTH_OK=true
|
||||||
STEP_AUTH_OK=true
|
STEP_AUTH_OK=true
|
||||||
AUTH_DEPLOYED=0
|
AUTH_DEPLOYED=0
|
||||||
@@ -375,8 +624,6 @@ SERVICES_FAILED=0
|
|||||||
ONBOARD_OK=false
|
ONBOARD_OK=false
|
||||||
ARR_SYNC_OK=false
|
ARR_SYNC_OK=false
|
||||||
WEBHOOK_SETUP_OK=false
|
WEBHOOK_SETUP_OK=false
|
||||||
MEDIA_SEED_OK=false
|
|
||||||
MEDIA_SEED_COUNT=0
|
|
||||||
WEBHOOK_LISTENER_OK=false
|
WEBHOOK_LISTENER_OK=false
|
||||||
MASTER_PUSH_OK=false
|
MASTER_PUSH_OK=false
|
||||||
|
|
||||||
@@ -443,15 +690,17 @@ if [[ "$PHASE1_ONLY" == true ]]; then
|
|||||||
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
|
||||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
||||||
|
LOCAL_SETUP_OK=false
|
||||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||||
|
fi
|
||||||
|
|
||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
|
||||||
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
|
||||||
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
echo " Conf push: skipped (needs SSH access to $MIRROR)"
|
||||||
echo " HOST1 setup: done ✅"
|
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
||||||
echo " Duration: $(format_duration $(( END - START )))"
|
echo " Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
echo " ACTION NEEDED: install the key on $MIRROR:"
|
echo " ACTION NEEDED: install the key on $MIRROR:"
|
||||||
@@ -482,11 +731,72 @@ if [[ "$PHASE1_ONLY" == true ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Phase 1 — Docker network on the mirror ────────────────────────────────────────────────
|
||||||
|
# Here, not only in Step 1b, because a --phase1-only run exits above and never reaches it.
|
||||||
|
# The mirror needs docker, not Varaverk, so this works before the plugin is installed — and
|
||||||
|
# creating the network now means it is in place long before the first container is deployed
|
||||||
|
# against it. Deploying against a missing network is what left twelve containers stuck in
|
||||||
|
# `Created`, so the earliest safe moment is the right one.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ Phase 1 — Docker Network ($MIRROR) ━━━"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would create any networks the stack templates reference on $MIRROR"
|
||||||
|
PHASE1_NET_OK=true
|
||||||
|
elif ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||||
|
PHASE1_NET_OK=true
|
||||||
|
else
|
||||||
|
warn "Network prep incomplete on $MIRROR — Step 1b retries this during Phase 2"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Phase 1 — Partner conf cache ──────────────────────────────────────────────────────────
|
||||||
|
# CONF_SYNC_ENABLED is armed here rather than waiting for Step 1d. It is the safe one of the
|
||||||
|
# three gates — it moves no data, it copies each side's host*.conf into the other's tmpfs so
|
||||||
|
# partner vars resolve — and the moment SSH works is the moment that should start. Leaving it
|
||||||
|
# until Phase 2 meant conf_sync.sh, which sources the conf fresh in its own process, exited
|
||||||
|
# on a closed gate every time it was called before then.
|
||||||
|
#
|
||||||
|
# Push always; pull only if the mirror actually has a conf yet. Before HOST2 installs Varaverk
|
||||||
|
# there is nothing to pull, and an unconditional pull would count a failure and notify about
|
||||||
|
# a condition that is simply "the mirror is not installed". The pull lands on the re-run after the
|
||||||
|
# install — the same --phase1-only --skip-ssh the operator uses to push conf again.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ Phase 1 — Partner Conf Cache ━━━"
|
||||||
|
_conf_sync_script="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would arm CONF_SYNC_ENABLED and cache confs with $MIRROR"
|
||||||
|
PHASE1_CACHE_OK=true
|
||||||
|
elif [[ ! -f "$_conf_sync_script" ]]; then
|
||||||
|
warn "conf_sync.sh not found — skipping partner conf cache"
|
||||||
|
else
|
||||||
|
set_conf_bool CONF_SYNC_ENABLED "true" "$SCRIPTS_ROOT/Configurations/master.conf" \
|
||||||
|
|| warn "Could not arm CONF_SYNC_ENABLED — cache step may no-op"
|
||||||
|
|
||||||
|
if bash "$_conf_sync_script" --push-only; then
|
||||||
|
PHASE1_CACHE_OK=true
|
||||||
|
else
|
||||||
|
warn "Could not push our conf to $MIRROR"
|
||||||
|
fi
|
||||||
|
|
||||||
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||||
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$MIRROR_IP" \
|
||||||
|
"[ -f '${_mirror_sd}/Configurations/${MIRROR_ID,,}.conf' ]" 2>/dev/null; then
|
||||||
|
bash "$_conf_sync_script" --pull-only || warn "Could not pull ${MIRROR_ID,,}.conf from $MIRROR"
|
||||||
|
else
|
||||||
|
echo " $MIRROR has no ${MIRROR_ID,,}.conf yet — it will cache once Varaverk is installed there"
|
||||||
|
fi
|
||||||
|
unset _mirror_sd
|
||||||
|
fi
|
||||||
|
unset _conf_sync_script
|
||||||
|
|
||||||
# HOST1 local setup — runs immediately without needing HOST2
|
# HOST1 local setup — runs immediately without needing HOST2
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
|
||||||
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
|
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
|
||||||
|
LOCAL_SETUP_OK=false
|
||||||
warn "Local setup had issues — check partnership_manager.sh output above"
|
warn "Local setup had issues — check partnership_manager.sh output above"
|
||||||
|
fi
|
||||||
|
|
||||||
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
|
||||||
|
|
||||||
@@ -495,11 +805,29 @@ if [[ "$PHASE1_ONLY" == true ]]; then
|
|||||||
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY PHASE 1 COMPLETE ━━━━━"
|
||||||
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
echo " SSH keys: $( [[ "$STEP_SSH_OK" == true ]] && echo "ready ✅" || echo "skipped" )"
|
||||||
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
|
||||||
echo " HOST1 setup: done ✅"
|
echo " Network: $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )"
|
||||||
|
echo " Conf cache: $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )"
|
||||||
|
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
|
||||||
echo " Duration: $(format_duration $(( END - START )))"
|
echo " Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."
|
||||||
echo " The wizard will detect the pushed conf and take the correct path."
|
# The push no longer waits for a Varaverk install on the far side. It resolves the partner's
|
||||||
|
# conf directory across both layouts and creates the internal one if neither exists, so the
|
||||||
|
# conf arrives BEFORE the plugin — which is the order that makes it useful. The .plg only
|
||||||
|
# seeds master.conf from the template when none is present, so what lands here survives the
|
||||||
|
# install and the wizard reads its identity straight out of it.
|
||||||
|
if [[ "$CONF_PUSH_OK" == true ]]; then
|
||||||
|
echo " master.conf is on $MIRROR — the wizard will find it and take the partner path,"
|
||||||
|
echo " already knowing $MY_ID and $MIRROR_ID. If the operator picks flash storage there,"
|
||||||
|
echo " storage_migrate.sh moves the conf to appdata with the rest of the install."
|
||||||
|
else
|
||||||
|
echo " master.conf was NOT delivered. Phase 1 seeds it into a bare host, so this is a"
|
||||||
|
echo " real failure, not the pre-install state — check SSH and that /boot is writable"
|
||||||
|
echo " on $MIRROR. Retry with:"
|
||||||
|
echo " • bash Partnership/partnership_onboard.sh --phase1-only --skip-ssh"
|
||||||
|
echo " • or push from Scheduler → master.conf → Save Conf"
|
||||||
|
echo " • or, once the plugin is installed, 'Pull from HOST1' on HOST2's Setup tab"
|
||||||
|
fi
|
||||||
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
echo " When HOST2 completes its onboard, it will automatically trigger Phase 2 here."
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
exit 0
|
exit 0
|
||||||
@@ -510,18 +838,45 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
|
echo "━━━ Step 1b — Docker Network (Mirror) ━━━"
|
||||||
|
|
||||||
_net_script="${SCRIPTS_ROOT}/Docker_Essentials/docker_network_connect.sh"
|
# Two halves, and the first is the one that matters for a fresh mirror.
|
||||||
|
#
|
||||||
|
# ensure_stack_networks_on_remote reads the networks out of the XMLs this onboard is about to
|
||||||
|
# push and creates any that are missing on the mirror. It does not consult the mirror's conf,
|
||||||
|
# because on a fresh node that array is the template default — a single commented-out entry —
|
||||||
|
# and an empty list is indistinguishable from "no networks needed". The result was every
|
||||||
|
# container in both stacks created against a network that did not exist.
|
||||||
|
#
|
||||||
|
# docker_network_connect.sh still runs afterwards: it is what *connects* the mirror's own
|
||||||
|
# listed containers to its own listed networks, which is a different job and remains the
|
||||||
|
# mirror's to declare.
|
||||||
|
if ! ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||||
|
warn "One or more stack networks could not be prepared on $MIRROR — deploys below may fail"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Resolve the path on the mirror, not here. This used SCRIPTS_ROOT — the OWNER's install path —
|
||||||
|
# to name a script it then runs over SSH on the MIRROR. That only holds while both hosts install
|
||||||
|
# to the same place, and an appdata-mode mirror is at /mnt/user/appdata/Varaverk, so bash was
|
||||||
|
# handed a path that does not exist there and Step 1b failed on every appdata install. The
|
||||||
|
# warning even printed the owner's path and told the operator to go check it on the mirror.
|
||||||
|
#
|
||||||
|
# Same fix as Steps 11 and 12, which already resolve the remote layout this way.
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
|
warn "DRY RUN — would run docker_network_connect.sh on $MIRROR"
|
||||||
STEP_NETWORK_OK=true
|
STEP_NETWORK_OK=true
|
||||||
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
|
|
||||||
"bash '$_net_script'" 2>/dev/null; then
|
|
||||||
echo "Docker network ready on $MIRROR ✅"
|
|
||||||
STEP_NETWORK_OK=true
|
|
||||||
else
|
else
|
||||||
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||||
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
_net_script="${_mirror_sd}/Docker_Essentials/docker_network_connect.sh"
|
||||||
|
if timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$MIRROR_IP" \
|
||||||
|
"[ -f '$_net_script' ] || { echo missing; exit 127; }; bash '$_net_script'" 2>/dev/null; then
|
||||||
|
echo "Docker network ready on $MIRROR ✅"
|
||||||
|
STEP_NETWORK_OK=true
|
||||||
|
else
|
||||||
|
warn "docker_network_connect.sh failed on $MIRROR — containers may fail if network is missing"
|
||||||
|
warn "Check ${_net_script} on $MIRROR and re-run with --skip-ssh if needed"
|
||||||
|
fi
|
||||||
|
unset _mirror_sd
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Step 1c: Share setup ─────────────────────────────────────────────────────────────────────
|
# ── Step 1c: Share setup ─────────────────────────────────────────────────────────────────────
|
||||||
@@ -536,6 +891,135 @@ else
|
|||||||
bash "$SCRIPT_DIR/share_setup.sh"
|
bash "$SCRIPT_DIR/share_setup.sh"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Step 1c2: Permit the mirror ───────────────────────────────────────────────────────────────
|
||||||
|
# A previous offboard blocklists the mirror, and rsync.sh refuses any host on that list outright
|
||||||
|
# — deliberately, so a stale cron cannot keep syncing to a partner that has been removed.
|
||||||
|
#
|
||||||
|
# The blocklist was cleared inside Step 8, which was fine while every rsync in the onboard came
|
||||||
|
# after it. Step 1e does not: it runs before the containers are deployed, which is the whole
|
||||||
|
# point of it. On a re-onboard that ordering means the provisioning sync is refused, the auth
|
||||||
|
# stack comes up against empty directories, and the step built to prevent exactly that reports
|
||||||
|
# a failure nobody can explain from its own output.
|
||||||
|
#
|
||||||
|
# Un-blocking here is not a loosening. The offboard's own summary says "re-onboard to permit
|
||||||
|
# access again" — an onboard is the event that permits it, and this is where the onboard starts
|
||||||
|
# needing it. Step 8 still runs its own clear; removing an absent entry is a no-op.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ Step 1c2 — Permit $MIRROR ━━━"
|
||||||
|
|
||||||
|
UNBLOCK_OK=true
|
||||||
|
_blocklist="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR}/partnership_blocklist.db}"
|
||||||
|
if [[ ! -f "$_blocklist" ]] || ! grep -q "^${MIRROR}|" "$_blocklist" 2>/dev/null; then
|
||||||
|
echo " $MIRROR is not blocklisted ✅"
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would remove $MIRROR from the partnership blocklist"
|
||||||
|
else
|
||||||
|
sed -i "/^${MIRROR}|/d" "$_blocklist" 2>/dev/null
|
||||||
|
if grep -q "^${MIRROR}|" "$_blocklist" 2>/dev/null; then
|
||||||
|
error "Could not remove $MIRROR from $_blocklist — rsync.sh will refuse it"
|
||||||
|
UNBLOCK_OK=false
|
||||||
|
else
|
||||||
|
echo " Removed $MIRROR from the blocklist ✅"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
unset _blocklist
|
||||||
|
|
||||||
|
# ── Step 1d: Sync gates ───────────────────────────────────────────────────────────────────────
|
||||||
|
# This was Step 9c, after the containers were already deployed. It has to run here instead,
|
||||||
|
# because Step 1e below is itself an rsync and Tier 1 stops every rsync — arming afterwards
|
||||||
|
# meant the provisioning sync exited cleanly having moved nothing.
|
||||||
|
#
|
||||||
|
# master.conf.template ships a fresh node inert. A successful Phase 1 is what makes Tier 1,
|
||||||
|
# CONF_SYNC and ARR_SYNC true; without this the defaults were a one-way door and somebody had
|
||||||
|
# to remember to hand-edit three toggles on the machine where forgetting looks exactly like
|
||||||
|
# everything working.
|
||||||
|
#
|
||||||
|
# Tier 2 is now set false rather than "left as configured". Tier 1 opening is what makes the
|
||||||
|
# scheduled jobs live, and a node that was onboarded ten seconds ago is not a node anyone has
|
||||||
|
# checked yet — free space, share layout, what the partner actually holds. Arriving connected
|
||||||
|
# and idle is the useful state; the operator arms each orchestrator when they mean to.
|
||||||
|
# MEDIA_SEED_ENABLED is not touched here at all — see Step 13.
|
||||||
|
#
|
||||||
|
# Owner only — the mirror receives these values in the Step 10 push rather than deciding.
|
||||||
|
ARM_OK=true
|
||||||
|
_VV_ARM_ON=(RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED)
|
||||||
|
_VV_ARM_OFF=(CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED
|
||||||
|
WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED)
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Step 1d — Sync Gates ━━━"
|
||||||
|
|
||||||
|
if [[ "$SKIP_ARM" == true ]]; then
|
||||||
|
echo " --no-arm — leaving the sync gates as they are"
|
||||||
|
warn " Step 1e needs Tier 1 open; with it closed the auth appdata will not be provisioned"
|
||||||
|
elif [[ "$AM_OWNER" != true ]]; then
|
||||||
|
echo " mirror — the owner's push decides these"
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would set ${_VV_ARM_ON[*]} true and ${_VV_ARM_OFF[*]} false"
|
||||||
|
else
|
||||||
|
_master_conf="$SCRIPTS_ROOT/Configurations/master.conf"
|
||||||
|
if [[ ! -f "$_master_conf" ]]; then
|
||||||
|
warn "master.conf not found at $_master_conf — gates left closed"
|
||||||
|
ARM_OK=false
|
||||||
|
else
|
||||||
|
cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)"
|
||||||
|
for _gate in "${_VV_ARM_ON[@]}"; do set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false; done
|
||||||
|
for _gate in "${_VV_ARM_OFF[@]}"; do set_conf_bool "$_gate" "false" "$_master_conf" || ARM_OK=false; done
|
||||||
|
unset _gate
|
||||||
|
echo " Tier 1 open: ${_VV_ARM_ON[*]}"
|
||||||
|
echo " Tier 2 off: every scheduled rsync — arm them yourself when you are ready"
|
||||||
|
fi
|
||||||
|
unset _master_conf
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Step 1e: Provision the auth stack's appdata ───────────────────────────────────────────────
|
||||||
|
# The one rsync an onboard actually needs, and it has to land before Step 3 creates the
|
||||||
|
# containers that read it. Deploying first meant Authelia, Lldap, NPM and both databases came up
|
||||||
|
# on the mirror against empty directories and initialised themselves from nothing — so the
|
||||||
|
# mirror's auth stack was a fresh install wearing the owner's container names.
|
||||||
|
#
|
||||||
|
# The critical-data profile, not critical-fallback. critical-data stops the auth containers on
|
||||||
|
# both sides for the duration, which costs the owner its auth for about a minute at 588 MB, and
|
||||||
|
# buys a consistent copy. critical-fallback is the dirty variant that keeps auth running, and a
|
||||||
|
# dirty copy is exactly what broke Redis and MariaDB on the mirror before: the manifest and the
|
||||||
|
# binlog index were copied without the files they name, and MariaDB hides that — the container
|
||||||
|
# reads Up while mysqld_safe restarts the dead database every few seconds.
|
||||||
|
#
|
||||||
|
# Media is not seeded here or anywhere in Phase 2. This share is 588 MB and the containers do
|
||||||
|
# not start correctly without it; a media library is terabytes and nothing waits on it.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_SYNC Step 1e — Auth Appdata Provision ━━━"
|
||||||
|
|
||||||
|
APPDATA_PROVISION_OK=false
|
||||||
|
if [[ "$SKIP_APPDATA_PROVISION" == true ]]; then
|
||||||
|
warn "Skipping (--skip-appdata-provision) — the mirror's auth stack will start empty"
|
||||||
|
elif [[ "${#PARTNERSHIP_PROVISION_SHARES[@]}" -eq 0 ]]; then
|
||||||
|
warn "PARTNERSHIP_PROVISION_SHARES is empty — nothing to provision"
|
||||||
|
warn " Set it in host${MY_ID: -1}.conf, or the mirror's auth stack starts from nothing"
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would sync ${PARTNERSHIP_PROVISION_SHARES[*]} to $MIRROR"
|
||||||
|
APPDATA_PROVISION_OK=true
|
||||||
|
else
|
||||||
|
_prov_rc=0
|
||||||
|
for _share in "${PARTNERSHIP_PROVISION_SHARES[@]}"; do
|
||||||
|
echo " Provisioning: $_share"
|
||||||
|
_prov_flags=()
|
||||||
|
[[ "$ENABLE_LOGGING" == true ]] && _prov_flags+=(--log)
|
||||||
|
if ! bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_prov_flags[@]}"; then
|
||||||
|
warn " Failed: $_share"
|
||||||
|
_prov_rc=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
unset _share _prov_flags
|
||||||
|
if [[ "$_prov_rc" -eq 0 ]]; then
|
||||||
|
echo "Auth appdata provisioned to $MIRROR ✅"
|
||||||
|
APPDATA_PROVISION_OK=true
|
||||||
|
else
|
||||||
|
warn "Auth appdata NOT fully provisioned — the containers below will start against"
|
||||||
|
warn " whatever is already on $MIRROR, which on a fresh node is nothing"
|
||||||
|
fi
|
||||||
|
unset _prov_rc
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
# ── Step 2: Stop mirror's existing auth stack ─────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━"
|
||||||
@@ -672,46 +1156,11 @@ else
|
|||||||
fi
|
fi
|
||||||
unset _webhook_script
|
unset _webhook_script
|
||||||
|
|
||||||
# ── Step 9c: Seed media shares on mirror ─────────────────────────────────────────────────────
|
# ── Step 9e: Start webhook listener on mirror ─────────────────────────────────────────────────
|
||||||
# arr_sync.sh bootstrapped the databases — mirror's arrs now know about all content but
|
|
||||||
# have no files on disk. Without this rsync, every imported item looks missing and arrs
|
|
||||||
# will immediately queue searches. --seed skips the empty-remote guard and does a clean push.
|
|
||||||
echo ""
|
|
||||||
echo "━━━ Step 9c — Media Share Seed ━━━"
|
|
||||||
|
|
||||||
if [[ "$SKIP_MEDIA_SEED" == true ]]; then
|
|
||||||
warn "Skipping (--skip-media-seed)"
|
|
||||||
elif [[ "${#DAILY_SYNC_SHARES[@]}" -eq 0 ]]; then
|
|
||||||
warn "DAILY_SYNC_SHARES empty for $MY_ID — skipping media seed"
|
|
||||||
warn "Configure HOST${MY_ID: -1}_DAILY_SYNC_SHARES in host${MY_ID: -1}.conf and run Rsync/rsync.sh --seed manually"
|
|
||||||
else
|
|
||||||
echo " Seeding ${#DAILY_SYNC_SHARES[@]} share(s) to $MIRROR — this may take a while"
|
|
||||||
_rsync_script="$SCRIPTS_ROOT/Rsync/rsync.sh"
|
|
||||||
_seed_flags=(--seed)
|
|
||||||
[[ "$DRY_RUN" == true ]] && _seed_flags+=(--dry-run)
|
|
||||||
[[ "$ENABLE_LOGGING" == true ]] && _seed_flags+=(--log)
|
|
||||||
for _share in "${DAILY_SYNC_SHARES[@]}"; do
|
|
||||||
echo " Seeding: $_share"
|
|
||||||
if bash "$_rsync_script" "$_share" "${_seed_flags[@]}"; then
|
|
||||||
(( MEDIA_SEED_COUNT++ )) || true
|
|
||||||
else
|
|
||||||
warn " Seed failed for $_share — re-run: Rsync/rsync.sh $_share --seed"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
if [[ "$MEDIA_SEED_COUNT" -gt 0 ]]; then
|
|
||||||
echo "Media seed complete — ${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} share(s) ✅"
|
|
||||||
MEDIA_SEED_OK=true
|
|
||||||
else
|
|
||||||
warn "Media seed: no shares completed — check errors above"
|
|
||||||
fi
|
|
||||||
unset _rsync_script _seed_flags _share
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ── Step 9d: Start webhook listener on mirror ─────────────────────────────────────────────────
|
|
||||||
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
|
# Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is
|
||||||
# already running — kick it now so events are captured immediately after onboard.
|
# already running — kick it now so events are captured immediately after onboard.
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Step 9d — Webhook Listener (Mirror) ━━━"
|
echo "━━━ Step 9e — Webhook Listener (Mirror) ━━━"
|
||||||
|
|
||||||
_listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh"
|
_listener_script="$SCRIPTS_ROOT/Arrs_Stack/start_webhook_listener.sh"
|
||||||
if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then
|
if [[ "$SKIP_WEBHOOK_LISTENER" == true ]]; then
|
||||||
@@ -756,6 +1205,133 @@ else
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Step 11: Service discovery on the mirror ──────────────────────────────────────────────────
|
||||||
|
# Deliberately last. conf_populate.sh fills host*.conf from what it can actually find running —
|
||||||
|
# arr API keys, container names, URLs — and until Step 3 and Step 5 deployed the auth and arr
|
||||||
|
# stacks there was nothing on the mirror to find. The wizard runs it during first-run setup,
|
||||||
|
# which on a fresh mirror is precisely the moment the machine is still empty, so everything it
|
||||||
|
# could have discovered was discovered as absent.
|
||||||
|
#
|
||||||
|
# No --overwrite: it only fills blanks, so anything the operator set by hand survives. --no-push
|
||||||
|
# because Step 10 above has just pushed conf; letting discovery push again would race it.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Step 11 — Service Discovery ($MIRROR) ━━━"
|
||||||
|
|
||||||
|
POPULATE_OK=false
|
||||||
|
# MIRROR_IP, not MIRROR_REACHABLE — the latter is partnership_offboard.sh's variable and does not
|
||||||
|
# exist in this script, so the test was always true against an empty string and Step 11 reported
|
||||||
|
# "skipped (unreachable)" on a mirror it had just deployed twelve containers to.
|
||||||
|
if [[ -z "${MIRROR_IP:-}" ]]; then
|
||||||
|
warn "$MIRROR has no resolved IP — skipping discovery, run Deployment/conf_populate.sh there later"
|
||||||
|
POPULATE_OK=skipped
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would run conf_populate.sh --no-push on $MIRROR"
|
||||||
|
POPULATE_OK=true
|
||||||
|
else
|
||||||
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||||
|
_pop_script="${_mirror_sd}/Deployment/conf_populate.sh"
|
||||||
|
if timeout 180 ssh -i "$MIRROR_SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$MIRROR_IP" \
|
||||||
|
"[ -f '$_pop_script' ] || { echo missing; exit 127; }; bash '$_pop_script' --no-push" 2>/dev/null; then
|
||||||
|
echo "Discovery complete on $MIRROR ✅"
|
||||||
|
POPULATE_OK=true
|
||||||
|
else
|
||||||
|
warn "Discovery failed on $MIRROR — run $_pop_script there by hand"
|
||||||
|
fi
|
||||||
|
unset _mirror_sd _pop_script
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Step 12: Group our containers on the mirror ───────────────────────────────────────────────
|
||||||
|
# The mirror now runs a dozen containers that are ours, scattered among its own. This files them
|
||||||
|
# under one folder named after us — "<OwnerShort>-Fallback" — matching the convention the owner
|
||||||
|
# already keeps for the mirror's containers.
|
||||||
|
#
|
||||||
|
# The icon is resolved HERE and passed over, not looked up there. It comes from the closest Emby
|
||||||
|
# user to our own name, and the mirror has neither our Emby key nor necessarily an Emby at all —
|
||||||
|
# so a lookup on that side would find nothing and the folder would come up blank.
|
||||||
|
#
|
||||||
|
# Not fatal in any direction: folder.view3 absent on the mirror is a clean skip, and a folder
|
||||||
|
# without a picture is still a folder.
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Step 12 — Container Grouping ($MIRROR) ━━━"
|
||||||
|
|
||||||
|
FOLDER_OK=false
|
||||||
|
if [[ -z "${MIRROR_IP:-}" ]]; then
|
||||||
|
warn "$MIRROR has no resolved IP — skipping container grouping"
|
||||||
|
FOLDER_OK=skipped
|
||||||
|
else
|
||||||
|
mapfile -t _deployed < <(deployed_stack_container_names)
|
||||||
|
_deployed_csv=$(IFS=,; echo "${_deployed[*]}")
|
||||||
|
if [[ -z "$_deployed_csv" ]]; then
|
||||||
|
log "No stack templates resolved to container names — nothing to group"
|
||||||
|
FOLDER_OK=skipped
|
||||||
|
elif [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would create ${MY_ID}-named fallback folder on $MIRROR with: $_deployed_csv"
|
||||||
|
FOLDER_OK=true
|
||||||
|
else
|
||||||
|
_ff_local="$SCRIPTS_ROOT/Plugin/$PLATFORM/Tools/fallback_folder.php"
|
||||||
|
_mf_local="$SCRIPTS_ROOT/Plugin/$PLATFORM/Tools/mirror_folders.php"
|
||||||
|
_icon=$(php "$_ff_local" --host="$MY_ID" --icon-only 2>/dev/null || true)
|
||||||
|
[[ -z "$_icon" ]] && log "No icon resolved for $MY_ID — folder will be created without one"
|
||||||
|
|
||||||
|
_mirror_sd=$(resolve_remote_scripts_dir "$MIRROR_IP" "$MIRROR_SSH_KEY" "no")
|
||||||
|
_ff_remote="${_mirror_sd}/Plugin/${PLATFORM}/Tools/fallback_folder.php"
|
||||||
|
_mf_remote="${_mirror_sd}/Plugin/${PLATFORM}/Tools/mirror_folders.php"
|
||||||
|
|
||||||
|
# ── 12a: reproduce our own folder layout on the mirror ────────────────────────────────
|
||||||
|
# The deployed stacks run on the mirror continuously — they are not failover coverage —
|
||||||
|
# so they belong on the same shelves they occupy here: Sonarr in "Arrs Stack", NPM and
|
||||||
|
# Lldap in "Networking", the databases in "Databases". Filing all of them under
|
||||||
|
# "<Owner>-Fallback", which is what this step used to do, records whose they are and
|
||||||
|
# nothing about what they do.
|
||||||
|
#
|
||||||
|
# The plan is computed here, where the owner's folder.view3 layout lives, and applied
|
||||||
|
# there. Whatever the layout does not account for comes back as "unfiled" and is what
|
||||||
|
# the fallback folder is actually for.
|
||||||
|
_unfiled="$_deployed_csv"
|
||||||
|
if [[ -f "$_mf_local" ]]; then
|
||||||
|
# Comma-joined: the flag takes a CSV, and "${arr[*]}" would join on spaces — which
|
||||||
|
# also happen to appear inside folder names, so the plan must never be reflowed.
|
||||||
|
_fbonly=$(IFS=,; printf '%s' "${PARTNERSHIP_FALLBACK_ONLY[*]:-}")
|
||||||
|
_plan=$(php "$_mf_local" --export --containers="$_deployed_csv" \
|
||||||
|
--fallback-only="$_fbonly" 2>/dev/null)
|
||||||
|
if [[ -n "$_plan" ]]; then
|
||||||
|
if printf '%s' "$_plan" | timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$MIRROR_IP" \
|
||||||
|
"[ -f '$_mf_remote' ] || exit 127; php '$_mf_remote' --import" 2>/dev/null; then
|
||||||
|
_unfiled=$(printf '%s' "$_plan" | php -r \
|
||||||
|
'echo implode(",", json_decode(stream_get_contents(STDIN),true)["unfiled"] ?? []);' 2>/dev/null)
|
||||||
|
else
|
||||||
|
warn "Could not mirror the folder layout to $MIRROR — falling back to one folder"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
unset _plan _fbonly
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 12b: the fallback folder gets only what is genuinely fallback ─────────────────────
|
||||||
|
# Unfiled containers plus anything named in PARTNERSHIP_FALLBACK_ONLY. Empty is the
|
||||||
|
# normal, correct outcome when every deployed container has a home in the layout.
|
||||||
|
if [[ -z "$_unfiled" ]]; then
|
||||||
|
echo " Folder layout mirrored — nothing left for ${MY_ID}-Fallback ✅"
|
||||||
|
FOLDER_OK=true
|
||||||
|
elif timeout 60 ssh -i "$MIRROR_SSH_KEY" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$MIRROR_IP" \
|
||||||
|
"[ -f '$_ff_remote' ] || { echo missing; exit 127; }
|
||||||
|
php '$_ff_remote' --host=$(printf '%q' "$MY_ID") \
|
||||||
|
--containers=$(printf '%q' "$_unfiled") \
|
||||||
|
--icon=$(printf '%q' "$_icon")" 2>/dev/null; then
|
||||||
|
FOLDER_OK=true
|
||||||
|
else
|
||||||
|
warn "Could not group containers on $MIRROR — run $_ff_remote there by hand"
|
||||||
|
fi
|
||||||
|
unset _ff_local _ff_remote _mf_local _mf_remote _mirror_sd _icon _unfiled
|
||||||
|
fi
|
||||||
|
unset _deployed _deployed_csv
|
||||||
|
fi
|
||||||
|
|
||||||
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
# ── Write Phase 2 completion state ────────────────────────────────────────────────────────────
|
||||||
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
[[ "$ONBOARD_OK" == true && "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 2
|
||||||
|
|
||||||
@@ -773,6 +1349,9 @@ _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
|||||||
|
|
||||||
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")"
|
||||||
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
|
echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")"
|
||||||
|
echo " Step 1c2— Permit mirror: $(_ok "$UNBLOCK_OK")"
|
||||||
|
echo " Step 1d — Sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$AM_OWNER" != true ]] && echo "mirror — set by owner" || _ok "$ARM_OK"; } )"
|
||||||
|
echo " Step 1e — Auth appdata: $( [[ "$SKIP_APPDATA_PROVISION" == true ]] && echo "skipped" || _ok "$APPDATA_PROVISION_OK" )"
|
||||||
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")"
|
||||||
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )"
|
||||||
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")"
|
||||||
@@ -782,9 +1361,10 @@ echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] &&
|
|||||||
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")"
|
||||||
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )"
|
||||||
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
|
echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")"
|
||||||
echo " Step 9c — Media seed: $( [[ "$SKIP_MEDIA_SEED" == true ]] && echo "skipped" || echo "${MEDIA_SEED_COUNT}/${#DAILY_SYNC_SHARES[@]} shares $(_ok "$MEDIA_SEED_OK")" )"
|
echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
||||||
echo " Step 9d — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")"
|
|
||||||
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )"
|
||||||
|
echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )"
|
||||||
|
echo " Step 12 — Grouping: $( [[ "$FOLDER_OK" == skipped ]] && echo "skipped" || _ok "$FOLDER_OK" )"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$ONBOARD_OK" == true ]]; then
|
if [[ "$ONBOARD_OK" == true ]]; then
|
||||||
|
|||||||
@@ -12,7 +12,13 @@
|
|||||||
#
|
#
|
||||||
# Key named after this server: hostname lowercased, unraid- prefix stripped.
|
# Key named after this server: hostname lowercased, unraid- prefix stripped.
|
||||||
# unRAID-Gmer4Lfe → gmer4lfe_rsync_automation
|
# unRAID-Gmer4Lfe → gmer4lfe_rsync_automation
|
||||||
# unRAID-Jayred365 → jayred365_rsync_automation
|
# unRAID-Jayred36 → jayred36_rsync_automation
|
||||||
|
#
|
||||||
|
# The name comes from the OS hostname, never from Tailscale. HOST2 answers to both
|
||||||
|
# `unRAID-Jayred36` (hostname -s — Unraid truncates Server Name to the 15-char NetBIOS limit)
|
||||||
|
# and `unraid-jayred365` (its Tailscale peer name), and only the first one decides this filename.
|
||||||
|
# api/setup.php derives the same path independently when it pulls master.conf, from the same
|
||||||
|
# source — so a key created under the Tailscale spelling is a key neither of them will find.
|
||||||
#
|
#
|
||||||
# Idempotent — skips generation if the key already exists (--force to regenerate).
|
# Idempotent — skips generation if the key already exists (--force to regenerate).
|
||||||
# Updates host*.conf with the key path on success.
|
# Updates host*.conf with the key path on success.
|
||||||
@@ -449,6 +455,22 @@ else
|
|||||||
if ssh-copy-id -i "$SSH_PUB_PATH" -o ConnectTimeout="${SSH_TIMEOUT:-15}" \
|
if ssh-copy-id -i "$SSH_PUB_PATH" -o ConnectTimeout="${SSH_TIMEOUT:-15}" \
|
||||||
root@"$REMOTE_SERVER" 2>/dev/null; then
|
root@"$REMOTE_SERVER" 2>/dev/null; then
|
||||||
echo "Public key installed on $REMOTE_SERVER_NAME ✅"
|
echo "Public key installed on $REMOTE_SERVER_NAME ✅"
|
||||||
|
|
||||||
|
# Tighten the file ssh-copy-id just wrote, or the key it installed is inert.
|
||||||
|
#
|
||||||
|
# On Unraid /root/.ssh is a symlink to /boot/config/ssh/root, and a file created there
|
||||||
|
# comes out 0666. sshd runs StrictModes by default and refuses to read a group- or
|
||||||
|
# world-writable authorized_keys — it logs "Authentication refused: bad ownership or
|
||||||
|
# modes" and falls through to password auth, which BatchMode then declines.
|
||||||
|
#
|
||||||
|
# The failure is silent from this end: ssh-copy-id succeeds and reports success, the key
|
||||||
|
# is present and byte-correct, and every later connection is still refused. Observed
|
||||||
|
# exactly that — key installed at 07:15:12, onboard refused at 07:15:43, blobs identical.
|
||||||
|
ssh -o ConnectTimeout="${SSH_TIMEOUT:-15}" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
-i "$SSH_KEY_PATH" root@"$REMOTE_SERVER" \
|
||||||
|
'chmod 700 "$(dirname "$(readlink -f ~/.ssh/authorized_keys)")" 2>/dev/null
|
||||||
|
chmod 600 "$(readlink -f ~/.ssh/authorized_keys)" 2>/dev/null' 2>/dev/null \
|
||||||
|
|| warn "Could not tighten authorized_keys on $REMOTE_SERVER_NAME — if auth is refused, chmod 600 it there"
|
||||||
else
|
else
|
||||||
error "ssh-copy-id failed — check that:"
|
error "ssh-copy-id failed — check that:"
|
||||||
error " 1. Remote server is reachable: tailscale status"
|
error " 1. Remote server is reachable: tailscale status"
|
||||||
|
|||||||
@@ -174,7 +174,12 @@ transform_xml_for_gpu() {
|
|||||||
local src_xml="$1" gpu_type="$2"
|
local src_xml="$1" gpu_type="$2"
|
||||||
|
|
||||||
# Detect GPU-aware XMLs — new style (--gpus "device=) or old style (--runtime=nvidia)
|
# Detect GPU-aware XMLs — new style (--gpus "device=) or old style (--runtime=nvidia)
|
||||||
if ! grep -qE '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
# -- before the pattern is load-bearing: it begins with "--", so without it grep parses the
|
||||||
|
# pattern as an OPTION, exits 2, and the ! makes this branch always true — the function then
|
||||||
|
# returned the XML untransformed every single time, for every GPU type, with 2>/dev/null
|
||||||
|
# swallowing "invalid option". Every container onboarded to a mirror kept the owner NVIDIA
|
||||||
|
# device UUID and could not start on Intel or AMD hardware. Found 2026-08-23.
|
||||||
|
if ! grep -qE -- '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
||||||
echo "$src_xml"
|
echo "$src_xml"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
@@ -279,7 +284,10 @@ deploy_container_from_xml() {
|
|||||||
[[ "$_transformed_xml" != "$xml_file" ]] && _gpu_tmp="$_transformed_xml"
|
[[ "$_transformed_xml" != "$xml_file" ]] && _gpu_tmp="$_transformed_xml"
|
||||||
xml_file="$_transformed_xml"
|
xml_file="$_transformed_xml"
|
||||||
|
|
||||||
local name repo network extra privileged
|
local name repo network extra privileged webui icon
|
||||||
|
# WebUI and Icon become Unraid labels below — see the docker create line for why.
|
||||||
|
webui=$( awk 'match($0,/<WebUI>([^<]*)<\/WebUI>/, a){print a[1];exit}' "$xml_file")
|
||||||
|
icon=$( awk 'match($0,/<Icon>([^<]*)<\/Icon>/, a){print a[1];exit}' "$xml_file")
|
||||||
name=$( awk 'match($0,/<Name>([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file")
|
name=$( awk 'match($0,/<Name>([^<]+)<\/Name>/, a){print a[1];exit}' "$xml_file")
|
||||||
repo=$( awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
|
repo=$( awk 'match($0,/<Repository>([^<]+)<\/Repository>/,a){print a[1];exit}' "$xml_file")
|
||||||
network=$( awk 'match($0,/<Network>([^<]+)<\/Network>/, a){print a[1];exit}' "$xml_file")
|
network=$( awk 'match($0,/<Network>([^<]+)<\/Network>/, a){print a[1];exit}' "$xml_file")
|
||||||
@@ -318,7 +326,19 @@ deploy_container_from_xml() {
|
|||||||
printf "docker stop %q 2>/dev/null || true\n" "$name"
|
printf "docker stop %q 2>/dev/null || true\n" "$name"
|
||||||
printf "docker rm %q 2>/dev/null || true\n" "$name"
|
printf "docker rm %q 2>/dev/null || true\n" "$name"
|
||||||
echo ""
|
echo ""
|
||||||
|
# Unraid's Docker Manager decides what it owns by label, not by template presence. The
|
||||||
|
# XML is SCPed to the mirror's templates-user above, but without these three the WebGUI
|
||||||
|
# lists the container as third-party: no Edit button, no WebUI link, no icon — the
|
||||||
|
# operator can see it running and cannot do anything with it.
|
||||||
|
#
|
||||||
|
# The values go in verbatim, placeholders and all: Unraid stores the literal
|
||||||
|
# "http://[IP]:[PORT:8989]/..." form in the label and substitutes at render time, so
|
||||||
|
# resolving them here would produce a link that stops being right the moment the
|
||||||
|
# container's port mapping changes.
|
||||||
printf "docker create --name %q --restart=unless-stopped" "$name"
|
printf "docker create --name %q --restart=unless-stopped" "$name"
|
||||||
|
printf " --label %q" "net.unraid.docker.managed=dockerman"
|
||||||
|
[[ -n "$webui" ]] && printf " --label %q" "net.unraid.docker.webui=${webui}"
|
||||||
|
[[ -n "$icon" ]] && printf " --label %q" "net.unraid.docker.icon=${icon}"
|
||||||
[[ -n "$network" ]] && printf " --network=%q" "$network"
|
[[ -n "$network" ]] && printf " --network=%q" "$network"
|
||||||
[[ "$privileged" == "true" ]] && printf " --privileged"
|
[[ "$privileged" == "true" ]] && printf " --privileged"
|
||||||
[[ -n "$extra" ]] && printf " %s" "$extra"
|
[[ -n "$extra" ]] && printf " %s" "$extra"
|
||||||
@@ -416,6 +436,112 @@ deploy_xml_stack() {
|
|||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── Ensure the networks our pushed templates reference exist on a remote host ─────────────────
|
||||||
|
#
|
||||||
|
# The owner deploys the mirror's containers from the owner's own XMLs, and those XMLs name a
|
||||||
|
# network. `docker create` fails outright if that network is missing, so the network has to
|
||||||
|
# exist on the mirror before any stack is deployed.
|
||||||
|
#
|
||||||
|
# This used to be left entirely to docker_network_connect.sh running on the mirror, which
|
||||||
|
# iterates the *mirror's* NETWORK_CONNECT_NETWORKS. host.conf.template ships that array with its
|
||||||
|
# only entry commented out, so on a fresh node it is empty — nothing was created, and every
|
||||||
|
# container in both stacks was created against a network that did not exist and could never
|
||||||
|
# start. Twelve containers stuck in `Created`, reported as "0 deployed, 8 failed" and
|
||||||
|
# "0 deployed, 5 failed" as though each container had its own problem.
|
||||||
|
#
|
||||||
|
# The owner knows what it is about to push, so it derives the requirement from the templates
|
||||||
|
# rather than trusting the mirror's conf to have been filled in.
|
||||||
|
#
|
||||||
|
# Only bridge networks are created. br0 and friends are ipvlan/macvlan bound to real host
|
||||||
|
# hardware — the parent interface cannot be inferred from here, and guessing one would attach
|
||||||
|
# the mirror's containers to the wrong segment.
|
||||||
|
#
|
||||||
|
# Usage: ensure_stack_networks_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||||
|
# ==============================================================================================
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── Container names this onboard deploys, read from the templates it deploys them from ────────
|
||||||
|
#
|
||||||
|
# Echoes one name per line. The <Name> element is the same value deploy_xml_stack() passes to
|
||||||
|
# `docker create --name`, so this is the deployed set by construction rather than by asking the
|
||||||
|
# mirror what it ended up with — which would also pick up whatever the mirror already ran.
|
||||||
|
#
|
||||||
|
# Usage: mapfile -t names < <(deployed_stack_container_names)
|
||||||
|
# ==============================================================================================
|
||||||
|
deployed_stack_container_names() {
|
||||||
|
local -a xml_names=()
|
||||||
|
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||||
|
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||||
|
[[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")
|
||||||
|
|
||||||
|
local xml_name xml_file cname
|
||||||
|
for xml_name in "${xml_names[@]}"; do
|
||||||
|
[[ -z "$xml_name" ]] && continue
|
||||||
|
xml_file="${TEMPLATES_DIR}/${xml_name}"
|
||||||
|
[[ -f "$xml_file" ]] || continue
|
||||||
|
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
||||||
|
[[ -n "$cname" ]] && echo "$cname"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
ensure_stack_networks_on_remote() {
|
||||||
|
local remote_ip="$1" ssh_key="$2"
|
||||||
|
local -a xml_names=() nets=()
|
||||||
|
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||||
|
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||||
|
[[ ${#PARTNERSHIP_SERVICES_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_SERVICES_STACK[@]}")
|
||||||
|
|
||||||
|
local xml_name xml_file net n seen
|
||||||
|
for xml_name in "${xml_names[@]}"; do
|
||||||
|
[[ -z "$xml_name" ]] && continue
|
||||||
|
xml_file="${TEMPLATES_DIR}/${xml_name}"
|
||||||
|
[[ -f "$xml_file" ]] || continue
|
||||||
|
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$xml_file" 2>/dev/null | head -1)
|
||||||
|
net="${net//[[:space:]]/}"
|
||||||
|
# Built-ins exist on every host; br* is host hardware, handled above.
|
||||||
|
case "$net" in ''|bridge|host|none|br[0-9]*) continue ;; esac
|
||||||
|
seen=false
|
||||||
|
for n in "${nets[@]}"; do [[ "$n" == "$net" ]] && { seen=true; break; }; done
|
||||||
|
[[ "$seen" == false ]] && nets+=("$net")
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${#nets[@]} -eq 0 ]]; then
|
||||||
|
log "No custom networks referenced by the pushed templates"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local rc=0 driver
|
||||||
|
for net in "${nets[@]}"; do
|
||||||
|
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" \
|
||||||
|
--format '{{.Driver}}' 2>/dev/null)
|
||||||
|
if [[ -z "$driver" ]]; then
|
||||||
|
warn "$net is referenced by a pushed template but does not exist here either — skipping"
|
||||||
|
rc=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "$driver" != "bridge" ]]; then
|
||||||
|
warn "$net is $driver here, not bridge — create it on $MIRROR by hand, its parent interface is host-specific"
|
||||||
|
rc=1
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would ensure network '$net' (bridge) exists on $MIRROR"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
|
||||||
|
root@"$remote_ip" \
|
||||||
|
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
||||||
|
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null; then
|
||||||
|
echo " network $net (bridge) ready on $MIRROR ✅"
|
||||||
|
else
|
||||||
|
warn " could not ensure network $net on $MIRROR — its containers will not start"
|
||||||
|
rc=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return "$rc"
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── Remove owner-deployed containers from a remote host ──────────────────────────────────────
|
# ── Remove owner-deployed containers from a remote host ──────────────────────────────────────
|
||||||
#
|
#
|
||||||
@@ -426,6 +552,7 @@ deploy_xml_stack() {
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
cleanup_deployed_stack_on_remote() {
|
cleanup_deployed_stack_on_remote() {
|
||||||
local remote_ip="$1" ssh_key="$2"
|
local remote_ip="$1" ssh_key="$2"
|
||||||
|
local _rc=0
|
||||||
local -a xml_names=()
|
local -a xml_names=()
|
||||||
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||||
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||||
@@ -471,13 +598,20 @@ cleanup_deployed_stack_on_remote() {
|
|||||||
|
|
||||||
while IFS= read -r path; do
|
while IFS= read -r path; do
|
||||||
[[ -z "$path" ]] && continue
|
[[ -z "$path" ]] && continue
|
||||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
if timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
|
||||||
echo " Appdata removed on $MIRROR: $path ✅" || \
|
echo " Appdata removed on $MIRROR: $path ✅"
|
||||||
|
else
|
||||||
warn " Failed to remove appdata on $MIRROR: $path"
|
warn " Failed to remove appdata on $MIRROR: $path"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
done <<< "$appdata_paths"
|
done <<< "$appdata_paths"
|
||||||
done
|
done
|
||||||
|
# Only appdata failures are counted. The container branch above cannot tell "removal failed"
|
||||||
|
# from "already gone" — both produce no `removed` echo — and an offboard re-run on a
|
||||||
|
# half-finished teardown is a normal case, so treating that as failure would cry wolf.
|
||||||
|
return "$_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -490,6 +624,12 @@ cleanup_deployed_stack_on_remote() {
|
|||||||
cleanup_deployed_stack_locally() {
|
cleanup_deployed_stack_locally() {
|
||||||
local owner_ip="$1" ssh_key="$2"
|
local owner_ip="$1" ssh_key="$2"
|
||||||
local -a xml_names=()
|
local -a xml_names=()
|
||||||
|
# Callers write `cleanup_deployed_stack_locally … || STEP_STACK_CLEANUP_OK=false`, so the
|
||||||
|
# exit status is what the offboard summary prints. Every removal below warns and carries on
|
||||||
|
# — one container that will not die must not abandon the rest of the stack — which meant the
|
||||||
|
# function ended on a `done` and could only ever return 0. Step 3 reported ✅ even when every
|
||||||
|
# docker rm and every rm -rf had failed. Failures are collected here and reported at the end.
|
||||||
|
local _rc=0
|
||||||
|
|
||||||
if [[ -n "$owner_ip" ]]; then
|
if [[ -n "$owner_ip" ]]; then
|
||||||
local -a auth_arr arr_arr
|
local -a auth_arr arr_arr
|
||||||
@@ -513,8 +653,13 @@ cleanup_deployed_stack_locally() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
||||||
log "Could not read deployed stack from owner — skipping auth/arr/services cleanup"
|
# Not a success. OWNER_REACHABLE only means a probe answered — the three SSH reads above
|
||||||
return 0
|
# can still time out or come back empty, and then nothing was cleaned. The caller's own
|
||||||
|
# unreachable-owner branch sets STEP_STACK_CLEANUP_OK=false for exactly this situation,
|
||||||
|
# so returning 0 here made the summary claim a cleanup that never ran.
|
||||||
|
warn "Could not read deployed stack from owner — auth/arr/services cleanup did not run"
|
||||||
|
warn "Containers will remain — re-run when the owner answers over SSH"
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local _local_short
|
local _local_short
|
||||||
@@ -551,17 +696,28 @@ cleanup_deployed_stack_locally() {
|
|||||||
"$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
"$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
||||||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
|
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
|
||||||
_PM_TRAP_STOPPED+=("$cname")
|
_PM_TRAP_STOPPED+=("$cname")
|
||||||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1 && \
|
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1; then
|
||||||
echo " $cname removed ✅" || warn " $cname rm failed"
|
echo " $cname removed ✅"
|
||||||
|
else
|
||||||
|
warn " $cname rm failed"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
log " $cname not found locally — skipping"
|
log " $cname not found locally — skipping"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
while IFS= read -r path; do
|
while IFS= read -r path; do
|
||||||
[[ -z "$path" ]] && continue
|
[[ -z "$path" ]] && continue
|
||||||
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
if rm -rf "$path"; then
|
||||||
|
echo " Appdata removed: $path ✅"
|
||||||
|
else
|
||||||
|
warn " Failed to remove: $path"
|
||||||
|
_rc=1
|
||||||
|
fi
|
||||||
done <<< "$appdata_paths"
|
done <<< "$appdata_paths"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
return "$_rc"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -578,11 +734,17 @@ reconfigure_webui() {
|
|||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Match <Name>Authelia</Name>, not "Authelia". Unraid writes the container name as bare XML
|
||||||
|
# text, so the quoted form matched nothing in any template on any host — which is why every
|
||||||
|
# offboard ended with four "template not found ... WebUI needs manual reconfiguration"
|
||||||
|
# warnings and left the mirror's auth WebUIs pointing at the owner it had just left.
|
||||||
|
#
|
||||||
|
# xargs -r so an empty first grep does not run the second one against the whole directory.
|
||||||
local template
|
local template
|
||||||
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||||
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
|
"grep -rl '<WebUI>' '$TEMPLATES_DIR/' 2>/dev/null | \
|
||||||
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
xargs -r grep -l '<Name>$container</Name>' 2>/dev/null | head -1" 2>/dev/null)
|
||||||
|
|
||||||
if [[ -z "$template" ]]; then
|
if [[ -z "$template" ]]; then
|
||||||
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
||||||
|
|||||||
@@ -103,7 +103,18 @@ once, and the endpoint, the cache writer, and the page all pick it up together.
|
|||||||
|-------|-------|------|
|
|-------|-------|------|
|
||||||
| `pages/` | 11 | One per WebGUI tab — monitor, docker, arrs, fallback, watchdog, rsync, partnership, scheduler, auth, settings, setup |
|
| `pages/` | 11 | One per WebGUI tab — monitor, docker, arrs, fallback, watchdog, rsync, partnership, scheduler, auth, settings, setup |
|
||||||
| `api/` | 50 | JSON endpoints the pages poll, plus action endpoints (run a script, stop a job, toggle a flag) |
|
| `api/` | 50 | JSON endpoints the pages poll, plus action endpoints (run a script, stop a job, toggle a flag) |
|
||||||
| `include/` | 16 | Shared builders and helpers — `vv_monitor_*`, `vv_arrs_*`, `vv_docker_*`, config read/write, auth |
|
| `include/` | 27 | Shared builders and helpers — `vv_monitor_*`, `vv_arrs_*`, `vv_docker_*`, config read/write, auth |
|
||||||
|
|
||||||
|
**The Monitor board is declared, not laid out.** `include/monitor_board.php` holds one array
|
||||||
|
naming every card on the Monitor grid, its width and its order, and generates the whole layout
|
||||||
|
from it — the column ladder, the span clamps at each width, the row-height cap, and the
|
||||||
|
compensation when a conditional card is absent. Column counts are 8/4/2/1 and spans are 1/2/4/8,
|
||||||
|
which is what lets the board re-cut itself at any width with no holes and no hand-placed card.
|
||||||
|
Breakpoints are arithmetic over `VV_MON_CARD_FLOOR`, never chosen by eye, and the row-height cap
|
||||||
|
is conditional on the same arithmetic — it applies only where a rung fits one screen.
|
||||||
|
`pages/monitor.php` carries the card bodies and nothing about where they go; moving a card is
|
||||||
|
moving a line in that array. The page cross-checks the declaration against the cards that
|
||||||
|
actually rendered and says so in the browser if they disagree.
|
||||||
|
|
||||||
**Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live
|
**Caching.** Several endpoints serve from `$VV_CACHE_DIR` (`/tmp/varaverk/api`, tmpfs) rather than hitting live
|
||||||
APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the
|
APIs on every page view, refreshed by `Tools/api_cache_writer.sh`. `?live=1` bypasses the
|
||||||
|
|||||||
@@ -24,6 +24,22 @@
|
|||||||
# list and reboot.
|
# list and reboot.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# One pass over PCIE_QUIET_DEVICES at array start. Each address is looked up under
|
||||||
|
# /sys/bus/pci/devices, checked against the guards below, and removed through the kernel's own
|
||||||
|
# per-device remove attribute. Nothing is masked and no kernel parameter is set.
|
||||||
|
#
|
||||||
|
# Idempotent, and silent when there is nothing to do. An address that is already gone is not an
|
||||||
|
# error — that is the normal state on every array start after the first within one boot.
|
||||||
|
#
|
||||||
|
# Reapplied every array start rather than once, because a reboot re-enumerates the bus and the
|
||||||
|
# devices come back. That is also the undo: clear PCIE_QUIET_DEVICES and reboot.
|
||||||
|
#
|
||||||
|
# PCIE_QUIET_ENABLED gates the whole run before any device is touched.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
Executable → Regular
+14
-14
@@ -18,6 +18,19 @@
|
|||||||
# total. Too high wastes RAM; too low causes slowdowns.
|
# total. Too high wastes RAM; too low causes slowdowns.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# 1. Read the current pm.max_children from the PHP-FPM pool config
|
||||||
|
# 2. Already at or above the target → exit silently, no write, no restart
|
||||||
|
# 3. Otherwise rewrite the value and restart PHP-FPM via the adapter
|
||||||
|
# 4. Verify PHP-FPM came back up
|
||||||
|
#
|
||||||
|
# Runs at array start, before the WebGUI sees real load. The setting does not survive an
|
||||||
|
# unRAID update — the OS replaces the pool config — which is why this reapplies every boot
|
||||||
|
# rather than being a one-time install step.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -41,19 +54,6 @@
|
|||||||
# 7. Read back config to confirm value applied
|
# 7. Read back config to confirm value applied
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
|
||||||
# ==============================================================================================
|
|
||||||
#
|
|
||||||
# 1. Read the current pm.max_children from the PHP-FPM pool config
|
|
||||||
# 2. Already at or above the target → exit silently, no write, no restart
|
|
||||||
# 3. Otherwise rewrite the value and restart PHP-FPM via the adapter
|
|
||||||
# 4. Verify PHP-FPM came back up
|
|
||||||
#
|
|
||||||
# Runs at array start, before the WebGUI sees real load. The setting does not survive an
|
|
||||||
# unRAID update — the OS replaces the pool config — which is why this reapplies every boot
|
|
||||||
# rather than being a one-time install step.
|
|
||||||
#
|
|
||||||
# ==============================================================================================
|
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -257,4 +257,4 @@ echo ""
|
|||||||
echo "$ICON_DONE Status: done ✅"
|
echo "$ICON_DONE Status: done ✅"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
Executable → Regular
+49
-22
@@ -14,6 +14,20 @@
|
|||||||
# page always reflects the live key value.
|
# page always reflects the live key value.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# 1. Check whether a Varaverk key already exists in the unraid-api registry
|
||||||
|
# 2. Create or overwrite it — the registry is ephemeral, so re-registering is the norm
|
||||||
|
# 3. Write the resulting key into this host's conf, replacing any previous value
|
||||||
|
# 4. Push the key into each partner's OWN conf, at the path their varaverk.cfg reports
|
||||||
|
# 5. Report whether the key was created, refreshed, or unchanged
|
||||||
|
#
|
||||||
|
# Runs at array start. The registry does not survive OS updates or an unraid-api restart,
|
||||||
|
# which is why this re-registers unconditionally rather than only when the key is missing —
|
||||||
|
# a key present in the conf but absent from the registry is the exact failure it repairs.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
# DESIGN PRINCIPLES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -28,19 +42,6 @@
|
|||||||
# without a separate sync step.
|
# without a separate sync step.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
|
||||||
# ==============================================================================================
|
|
||||||
#
|
|
||||||
# 1. Check whether a Varaverk key already exists in the unraid-api registry
|
|
||||||
# 2. Create or overwrite it — the registry is ephemeral, so re-registering is the norm
|
|
||||||
# 3. Write the resulting key into this host's conf, replacing any previous value
|
|
||||||
# 4. Report whether the key was created, refreshed, or unchanged
|
|
||||||
#
|
|
||||||
# Runs at array start. The registry does not survive OS updates or an unraid-api restart,
|
|
||||||
# which is why this re-registers unconditionally rather than only when the key is missing —
|
|
||||||
# a key present in the conf but absent from the registry is the exact failure it repairs.
|
|
||||||
#
|
|
||||||
# ==============================================================================================
|
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -49,6 +50,17 @@
|
|||||||
# Conf file check — aborts before any writes if the host conf is missing
|
# Conf file check — aborts before any writes if the host conf is missing
|
||||||
# dry-run mode — shows what would happen without touching anything
|
# dry-run mode — shows what would happen without touching anything
|
||||||
#
|
#
|
||||||
|
# Remote path discovery
|
||||||
|
# The partner's conf path comes from resolve_remote_scripts_dir(), which reads their
|
||||||
|
# varaverk.cfg, so a partner in appdata storage mode is found. The path was hardcoded
|
||||||
|
# to the flash plugin directory, which is wrong for any such partner.
|
||||||
|
#
|
||||||
|
# Remote target must exist
|
||||||
|
# The pushed script refuses to create the conf and reports the path it looked at.
|
||||||
|
# resolve_remote_scripts_dir() falls back to our own SCRIPTS_DIR when the probe fails,
|
||||||
|
# and appending an API key to a merely plausible path is how the hardcoded version
|
||||||
|
# failed without saying so.
|
||||||
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -194,6 +206,9 @@ if [[ -z "$SSH_KEY" ]]; then
|
|||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# resolve_remote_scripts_dir() reads this; every inline timeout below already uses 10.
|
||||||
|
SSH_TIMEOUT=10
|
||||||
|
|
||||||
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
|
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
|
||||||
partner_host="${!host_var}"
|
partner_host="${!host_var}"
|
||||||
[[ -z "$partner_host" ]] && continue
|
[[ -z "$partner_host" ]] && continue
|
||||||
@@ -203,16 +218,27 @@ for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
|
|||||||
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
|
||||||
[[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
|
[[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
|
||||||
|
|
||||||
# Target is the partner's OWN conf on their machine
|
# Target is the partner's OWN conf on their machine. Their SCRIPTS_DIR is read from their
|
||||||
partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
|
# varaverk.cfg rather than assumed — this was hardcoded to the flash plugin path, so a
|
||||||
|
# partner in appdata storage mode had its key appended under a directory that does not
|
||||||
|
# exist there. HOST2 has run in appdata mode since it was installed.
|
||||||
|
partner_sd=$(resolve_remote_scripts_dir "$partner_ip" "$SSH_KEY" "no")
|
||||||
|
partner_conf="${partner_sd}/Configurations/${partner_slot}.conf"
|
||||||
tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
|
tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
|
||||||
remote="/tmp/vv_kp_${RANDOM}.sh"
|
remote="/tmp/vv_kp_${RANDOM}.sh"
|
||||||
chmod 700 "$tmp"
|
chmod 700 "$tmp"
|
||||||
|
|
||||||
# Key stays in the temp file — never appears in SSH command args
|
# Key stays in the temp file — never appears in SSH command args.
|
||||||
|
# The conf must already exist: resolve_remote_scripts_dir falls back to our own SCRIPTS_DIR
|
||||||
|
# when the probe fails, and appending a key to a path that is merely plausible is how the
|
||||||
|
# hardcoded version failed silently. Report the path instead of guessing.
|
||||||
cat > "$tmp" <<PUSHSCRIPT
|
cat > "$tmp" <<PUSHSCRIPT
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
target='${partner_conf}'
|
target='${partner_conf}'
|
||||||
|
if [ ! -f "\$target" ]; then
|
||||||
|
echo "missing:\$target"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
|
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
|
||||||
sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
|
sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
|
||||||
else
|
else
|
||||||
@@ -223,13 +249,14 @@ PUSHSCRIPT
|
|||||||
|
|
||||||
if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||||
-o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
|
-o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
|
||||||
if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
push_out=$(timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
|
||||||
-o StrictHostKeyChecking=no "root@${partner_ip}" \
|
-o StrictHostKeyChecking=no "root@${partner_ip}" \
|
||||||
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
|
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null)
|
||||||
echo "Key pushed to $partner_host ✅"
|
case "$push_out" in
|
||||||
else
|
*ok*) echo "Key pushed to $partner_host ✅" ;;
|
||||||
warn "Key push to $partner_host failed — they can create their own copy"
|
missing:*) warn "Key push to $partner_host failed — no conf at ${push_out#missing:}" ;;
|
||||||
fi
|
*) warn "Key push to $partner_host failed — they can create their own copy" ;;
|
||||||
|
esac
|
||||||
else
|
else
|
||||||
warn "SCP to $partner_host failed — skipping"
|
warn "SCP to $partner_host failed — skipping"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -122,8 +122,17 @@ if ($explain) {
|
|||||||
exit(2);
|
exit(2);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
[$jobFile, $question, $historyJson, $kind, $think, $profile, $scope, $webArg] =
|
// Ninth argument is the node that asked, which is not necessarily the node generating: the
|
||||||
array_slice($argv, 1, 8) + array_fill(0, 8, '');
|
// mesh shares one AI, so a turn started on a mirror runs here. Without it every row in the
|
||||||
|
// ledger reads as the owner's and the tab can no longer say who spent what.
|
||||||
|
//
|
||||||
|
// Optional, and validated rather than trusted. It arrives from the RPC layer's own view of
|
||||||
|
// the caller, but this is a command line and an unrecognisable value must fall back to "this
|
||||||
|
// node" rather than write a slot name nobody can place.
|
||||||
|
[$jobFile, $question, $historyJson, $kind, $think, $profile, $scope, $webArg, $askNode] =
|
||||||
|
array_slice($argv, 1, 9) + array_fill(0, 9, '');
|
||||||
|
|
||||||
|
if (!preg_match('/^host\d+$/', (string)$askNode)) $askNode = '';
|
||||||
|
|
||||||
if ($jobFile === '' || $question === '') exit(1);
|
if ($jobFile === '' || $question === '') exit(1);
|
||||||
if (!preg_match('#/[0-9a-f]{32}\.json$#', $jobFile)) exit(1);
|
if (!preg_match('#/[0-9a-f]{32}\.json$#', $jobFile)) exit(1);
|
||||||
@@ -413,6 +422,17 @@ if ($can('system_state')) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fallback is dormant until it isn't, so "configured" and "would work" are unrelated — this is the
|
||||||
|
// only block that reports the second. Cheap: local conf and state file plus one cached presence
|
||||||
|
// read, never a network round trip.
|
||||||
|
if ($can('fallback_state')) {
|
||||||
|
$fb = vv_ai_fallback_state();
|
||||||
|
if ($fb !== '') {
|
||||||
|
$attached['fallback'] = 'readiness';
|
||||||
|
$diagBlock .= $fb;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The troubleshooting profile gets the actual tail of the one log the operator is looking at,
|
// The troubleshooting profile gets the actual tail of the one log the operator is looking at,
|
||||||
// warnings and ordinary lines alike. The fleet-wide WARN/ERROR sweep above cannot answer "why
|
// warnings and ordinary lines alike. The fleet-wide WARN/ERROR sweep above cannot answer "why
|
||||||
// did this one stop" — the last line a script printed before dying is usually not labelled.
|
// did this one stop" — the last line a script printed before dying is usually not labelled.
|
||||||
@@ -915,6 +935,32 @@ if ($profile === 'chat' && vv_ai_chat_needs_varaverk($question)) {
|
|||||||
|
|
||||||
if ($context !== '') $system .= "PASSAGES\n" . $context;
|
if ($context !== '') $system .= "PASSAGES\n" . $context;
|
||||||
|
|
||||||
|
// What the transcript can render, offered rather than required.
|
||||||
|
//
|
||||||
|
// Appended to every profile at the single point they all pass through, so a new profile cannot
|
||||||
|
// be added without it. The set is the one the mesh composer already offers, and the renderer
|
||||||
|
// maps colour NAMES to fixed values — the model never writes a colour, only chooses one, so
|
||||||
|
// nothing it emits can become a style property.
|
||||||
|
//
|
||||||
|
// Deliberately permissive in tone. The interesting question is whether it reaches for these at
|
||||||
|
// all and where; an instruction to use them would answer that question by force and tell us
|
||||||
|
// nothing. Prose stays prose.
|
||||||
|
$system .= "\n\nFORMATTING AVAILABLE TO YOU\n"
|
||||||
|
. "The transcript renders a small set of marks. Use them where they genuinely help a "
|
||||||
|
. "reader — a warning that must not be skimmed, a value worth picking out of a "
|
||||||
|
. "sentence — and not otherwise. Plain prose is the default and needs no marking.\n"
|
||||||
|
. " **bold** emphasis\n"
|
||||||
|
. " *italic* aside or term\n"
|
||||||
|
. " __underline__ rarely; reserve for something that must not be missed\n"
|
||||||
|
. " {big}…{/big} one size up\n"
|
||||||
|
. " {small}…{/small} one size down, for asides\n"
|
||||||
|
. " {green}…{/green} also blue, amber, red, violet, grey\n"
|
||||||
|
. "Colour carries meaning here: green for confirmed-good, amber for caution, red for "
|
||||||
|
. "something broken or destructive. Do not colour text decoratively — a coloured word "
|
||||||
|
. "that means nothing costs the colours their meaning everywhere else. Code fences and "
|
||||||
|
. "`inline code` work as before and remain the right choice for anything typed at a "
|
||||||
|
. "shell.\n";
|
||||||
|
|
||||||
$messages = [['role' => 'system', 'content' => $system]];
|
$messages = [['role' => 'system', 'content' => $system]];
|
||||||
$hist = json_decode($historyJson ?: '[]', true);
|
$hist = json_decode($historyJson ?: '[]', true);
|
||||||
if (is_array($hist)) {
|
if (is_array($hist)) {
|
||||||
@@ -1213,7 +1259,8 @@ $tokS = $evalNs > 0 ? round($evalCount / ($evalNs / 1e9), 1) : null;
|
|||||||
|
|
||||||
// Accounting before the job file is written, so a turn is counted even if the tab has already
|
// Accounting before the job file is written, so a turn is counted even if the tab has already
|
||||||
// been closed and nobody ever reads the result. Best-effort by contract — it cannot throw.
|
// been closed and nobody ever reads the result. Best-effort by contract — it cannot throw.
|
||||||
vv_ai_token_record($profile, 'webgui', (int)($d['prompt_eval_count'] ?? 0), $evalCount, $tokS);
|
vv_ai_token_record($profile, 'webgui', (int)($d['prompt_eval_count'] ?? 0), $evalCount, $tokS,
|
||||||
|
$askNode ?? '');
|
||||||
|
|
||||||
// profile is the one that actually answered, not the one that was asked for — they differ on a
|
// profile is the one that actually answered, not the one that was asked for — they differ on a
|
||||||
// handoff. Reported so the page can show which contract produced the answer rather than the
|
// handoff. Reported so the page can show which contract produced the answer rather than the
|
||||||
|
|||||||
@@ -16,12 +16,7 @@
|
|||||||
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
|
# Not scheduled and deliberately not in any orchestrator. This is a development check — it runs
|
||||||
# when the routing changes, not every night. Nothing on the running system depends on it.
|
# when the routing changes, not every night. Nothing on the running system depends on it.
|
||||||
#
|
#
|
||||||
# RUNTIME MODES
|
# DESIGN PRINCIPLES
|
||||||
# ai_explain_check.sh check every fixture
|
|
||||||
# ai_explain_check.sh --verbose print the full explain report for each case
|
|
||||||
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
|
|
||||||
#
|
|
||||||
# OPERATIONAL SAFEGUARDS
|
|
||||||
# Asserts routing, never wording.
|
# Asserts routing, never wording.
|
||||||
# Which capabilities a profile holds and which evidence was attached are decided before the
|
# Which capabilities a profile holds and which evidence was attached are decided before the
|
||||||
# model is asked anything. Asserting on generated prose would fail for reasons that tell
|
# model is asked anything. Asserting on generated prose would fail for reasons that tell
|
||||||
@@ -31,8 +26,16 @@
|
|||||||
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
|
# An unrecognised key is an error, not a skip. A typo in an assertion that silently passes
|
||||||
# is worse than no assertion, because the line still reads as covered.
|
# is worse than no assertion, because the line still reads as covered.
|
||||||
#
|
#
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# Read-only. Runs fixtures through --explain and compares; writes nothing and changes no state.
|
||||||
|
#
|
||||||
# Exits non-zero on any failure, so it can gate a commit.
|
# Exits non-zero on any failure, so it can gate a commit.
|
||||||
#
|
#
|
||||||
|
# RUNTIME MODES
|
||||||
|
# ai_explain_check.sh check every fixture
|
||||||
|
# ai_explain_check.sh --verbose print the full explain report for each case
|
||||||
|
# ai_explain_check.sh <pattern> only cases whose question matches the pattern
|
||||||
|
#
|
||||||
# DEPENDS ON
|
# DEPENDS ON
|
||||||
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
|
# Plugin/unraid/Tools/ai_chat_worker.php --explain mode
|
||||||
# Plugin/unraid/Tools/ai_explain_fixtures.txt
|
# Plugin/unraid/Tools/ai_explain_fixtures.txt
|
||||||
|
|||||||
@@ -8,10 +8,34 @@
|
|||||||
// Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing,
|
// Reads ai_log_fixtures.txt and this host's /var/log/syslog*. Files nothing, writes nothing,
|
||||||
// and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself.
|
// and calls no part of the sweep beyond vv_ai_syslog_findings() on lines it supplies itself.
|
||||||
//
|
//
|
||||||
// EXIT
|
// DESIGN PRINCIPLES
|
||||||
// 0 when every fixture is recognised as written. Precision findings are reported but never
|
// Only recall can fail the run.
|
||||||
// fail the run: what a real syslog contains is a fact about the machine, not about the
|
// A missed fixture is a fact about the patterns and is always a defect. A precision hit is
|
||||||
// patterns, and a genuinely failing disk should not turn this into a red test.
|
// a fact about this machine — a genuinely failing disk should not turn this red, and if it
|
||||||
|
// did, the honest fix would be to stop having a failing disk rather than to edit a pattern.
|
||||||
|
//
|
||||||
|
// Precision is replayed against real history, not a sample.
|
||||||
|
// The patterns that cause damage are the ones matching ordinary operation, and ordinary
|
||||||
|
// operation is exactly what a hand-written fixture file never contains. Only the machine's
|
||||||
|
// own syslog can show what a pattern fires on when nothing is wrong.
|
||||||
|
//
|
||||||
|
// The sweep is never invoked, only its matcher.
|
||||||
|
// vv_ai_syslog_findings() is called on lines this file supplies. Running the real sweep
|
||||||
|
// would file findings, and a test that has to be cleaned up afterwards stops being run.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Read-only. Reads ai_log_fixtures.txt and this host's /var/log/syslog*; files no finding,
|
||||||
|
// writes no store, and touches no conf beyond the enable flag.
|
||||||
|
//
|
||||||
|
// Exit 0 when every fixture is recognised as written. Precision findings are reported but
|
||||||
|
// never fail the run — see DESIGN PRINCIPLES.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// php ai_log_check.php both checks
|
||||||
|
// php ai_log_check.php --recall fixtures only
|
||||||
|
// php ai_log_check.php --precision replay this host's syslog history only
|
||||||
|
//
|
||||||
|
// Not scheduled, and deliberately so. Run it after touching VV_AI_SYSLOG_PATTERNS.
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,47 @@
|
|||||||
#
|
#
|
||||||
# Run it after touching VV_AI_SYSLOG_PATTERNS. Nothing here writes: no findings are filed, no
|
# Run it after touching VV_AI_SYSLOG_PATTERNS. Nothing here writes: no findings are filed, no
|
||||||
# conf is read for anything but the enable flag, and the sweep is never invoked.
|
# conf is read for anything but the enable flag, and the sweep is never invoked.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# A wrapper. The work is in ai_log_check.php, next to the sweep's own matcher — the patterns and
|
||||||
|
# vv_ai_syslog_findings() live in include/ai_repair.php, and a bash reimplementation of the
|
||||||
|
# matching would be a second set of regexes to keep in step with the first.
|
||||||
|
#
|
||||||
|
# Flags are forwarded verbatim; nothing is interpreted here.
|
||||||
|
#
|
||||||
|
# Not scheduled and in no orchestrator. This is a development check that runs when the patterns
|
||||||
|
# change, not on a timer — nothing on the running system depends on it.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Two checks, because the failure modes are opposite.
|
||||||
|
# Recall catches a pattern that stopped matching; precision catches one that matches too much.
|
||||||
|
# A single test would catch one and silently permit the other, and the second is the one that
|
||||||
|
# fills the findings store with noise until the operator stops reading it.
|
||||||
|
#
|
||||||
|
# Precision is measured against this machine's real history.
|
||||||
|
# A hand-written fixture file can show that a pattern matches what it should. Only a real
|
||||||
|
# syslog can show what it also matches when nothing is wrong.
|
||||||
|
#
|
||||||
|
# Only recall fails the run.
|
||||||
|
# What a real syslog contains is a fact about the machine, not about the patterns. A genuinely
|
||||||
|
# failing disk should not turn this red.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Read-only. No finding is filed, no store is written, and the repair sweep itself is never run
|
||||||
|
# — only its matcher, on lines this check supplies.
|
||||||
|
#
|
||||||
|
# Safe to run on a live host at any time, including one that is currently faulting. It observes
|
||||||
|
# the syslog it replays and changes nothing about it.
|
||||||
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// The mesh entry point for the AI subsystem. Another node forwards an action here over SSH; this
|
||||||
|
// runs it through the same vv_ai_dispatch() a browser request on this node would have used, and
|
||||||
|
// writes the response back as JSON on stdout.
|
||||||
|
//
|
||||||
|
// CLI only. It is never served over HTTP and takes nothing from the environment but stdin.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// stdin {"action":"ask","params":{…},"is_post":true}
|
||||||
|
// stdout {"_vv_rpc":1,"status":200,"body":{…}}
|
||||||
|
//
|
||||||
|
// The wrapper exists so an HTTP status can travel with the body — a 405 raised in the dispatcher
|
||||||
|
// has to arrive at the calling node as a 405, not as a 200 carrying an error string.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// One implementation, two doors.
|
||||||
|
// This shares every handler with api/ai.php through include/ai_actions.php. A mesh request
|
||||||
|
// and a browser request cannot diverge in behaviour because there is only one behaviour.
|
||||||
|
//
|
||||||
|
// The caller's node id is a label, not a claim.
|
||||||
|
// params._vv_node says which node asked, so findings and incidents are stored against the
|
||||||
|
// node they describe. It is not consulted for authorization — possession of the partnership
|
||||||
|
// SSH key already settled that, at onboard.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Refuses to run anywhere but the AI owner.
|
||||||
|
// A mirror that somehow received a forwarded request must not answer it from its own empty
|
||||||
|
// stores. Exits non-zero so the caller reports a transport failure rather than rendering an
|
||||||
|
// empty success.
|
||||||
|
//
|
||||||
|
// Refuses to run over the web.
|
||||||
|
// Guarded on PHP_SAPI. Reachable over HTTP this would be an unauthenticated bypass of every
|
||||||
|
// gate api/ai.php applies, since it takes its whole request from stdin.
|
||||||
|
//
|
||||||
|
// The master switch is honoured here too.
|
||||||
|
// AI_ENABLED false on the owner means the mesh gets the same refusal a local request gets.
|
||||||
|
// The calling node checks its own switch; this one checks the owner's.
|
||||||
|
//
|
||||||
|
// Input is size-capped before it is decoded.
|
||||||
|
// stdin is an untrusted stream from another process. A malformed or endless payload must
|
||||||
|
// fail as a bad request, not as an out-of-memory.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// Not invoked by hand. include/ai_rpc.php opens an SSH session to the owner and runs this
|
||||||
|
// file with the request as JSON on stdin; the response is JSON on stdout. There are no flags
|
||||||
|
// and no arguments — the action, the profile and the payload all arrive in the request body.
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/ai_actions.php vv_ai_dispatch() — the shared handlers
|
||||||
|
// include/config.php vv_ai_is_owner()
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
if (PHP_SAPI !== 'cli') {
|
||||||
|
http_response_code(404);
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
require_once dirname(__DIR__) . '/include/ai_actions.php';
|
||||||
|
|
||||||
|
function vv_rpc_out(int $status, array $body): void {
|
||||||
|
echo json_encode(['_vv_rpc' => 1, 'status' => $status, 'body' => $body],
|
||||||
|
JSON_UNESCAPED_SLASHES), "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not this node's job. Exit non-zero: the caller must see a transport failure, not an answer
|
||||||
|
// assembled from stores that are empty here by design.
|
||||||
|
if (!vv_ai_is_owner()) {
|
||||||
|
fwrite(STDERR, "ai_rpc: this node is not the AI owner\n");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1 MiB. A turn's history is the largest legitimate payload and is capped far below this by the
|
||||||
|
// per-message truncation in the dispatcher; anything larger is not a request this serves.
|
||||||
|
$raw = stream_get_contents(STDIN, 1024 * 1024);
|
||||||
|
$req = json_decode((string)$raw, true);
|
||||||
|
if (!is_array($req)) {
|
||||||
|
vv_rpc_out(400, ['ok' => false, 'error' => 'ai_rpc: unreadable request']);
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
$action = trim((string)($req['action'] ?? ''));
|
||||||
|
$params = is_array($req['params'] ?? null) ? $req['params'] : [];
|
||||||
|
$isPost = (bool)($req['is_post'] ?? false);
|
||||||
|
|
||||||
|
if ($action === '') {
|
||||||
|
vv_rpc_out(400, ['ok' => false, 'error' => 'ai_rpc: no action']);
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// The owner's own switch. The calling node already checked its own; this is the other half, and
|
||||||
|
// it is what makes turning AI off here take it off the whole mesh.
|
||||||
|
if (!vv_ai_enabled()) {
|
||||||
|
vv_rpc_out(200, ['ok' => false, 'error' => 'AI_ENABLED is false on the AI owner — AI features are off']);
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
vv_ai_log(sprintf('rpc action=%s from=%s', $action, (string)($params['_vv_node'] ?? '?')));
|
||||||
|
|
||||||
|
$httpStatus = 200;
|
||||||
|
$body = vv_ai_dispatch($action, $params, $isPost, $httpStatus);
|
||||||
|
vv_rpc_out($httpStatus, $body);
|
||||||
@@ -66,6 +66,13 @@
|
|||||||
// VV_CACHE_DIR/arrs.json consumed by api/arrs.php
|
// VV_CACHE_DIR/arrs.json consumed by api/arrs.php
|
||||||
// stdout one timing line, captured into the job log
|
// stdout one timing line, captured into the job log
|
||||||
//
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// api_cache_writer.php
|
||||||
|
// Build both payloads and write them to the cache. Takes no arguments.
|
||||||
|
//
|
||||||
|
// Run every minute by Tools/api_cache_writer.sh, which is the cron entry — this file is the
|
||||||
|
// work, that one is the schedule, the lock and the log.
|
||||||
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/monitor.php, include/common.php, include/unraid_api.php,
|
// include/monitor.php, include/common.php, include/unraid_api.php,
|
||||||
// include/vms.php, include/docker_folders.php, include/arrs.php
|
// include/vms.php, include/docker_folders.php, include/arrs.php
|
||||||
@@ -112,6 +119,7 @@ if (vv_ai_ui_on()) {
|
|||||||
|
|
||||||
$monitor = [
|
$monitor = [
|
||||||
'system' => vv_system_info(),
|
'system' => vv_system_info(),
|
||||||
|
'varaverk' => vv_varaverk_state(),
|
||||||
'fallback' => vv_fallback_state(),
|
'fallback' => vv_fallback_state(),
|
||||||
'fallback_active' => vv_fallback_active(),
|
'fallback_active' => vv_fallback_active(),
|
||||||
'partner' => vv_partner_state(),
|
'partner' => vv_partner_state(),
|
||||||
@@ -149,7 +157,7 @@ vv_cache_write('arrs', $arrs);
|
|||||||
|
|
||||||
// ── Watchdog payload ──────────────────────────────────────────────────────────
|
// ── Watchdog payload ──────────────────────────────────────────────────────────
|
||||||
// Here for the same reason as the other two: vv_wd_all() SSHes to every partner, which cost 8.3s
|
// Here for the same reason as the other two: vv_wd_all() SSHes to every partner, which cost 8.3s
|
||||||
// on this host with HOST2 down, and the Watchdog tab polls every 30 seconds. Collected off the
|
// on this host with the partner unreachable, and the Watchdog tab polls every 30 seconds. Collected off the
|
||||||
// request path it is paid once a minute by cron instead of by whoever has the tab open.
|
// request path it is paid once a minute by cron instead of by whoever has the tab open.
|
||||||
require_once $_base . '/include/watchdog.php';
|
require_once $_base . '/include/watchdog.php';
|
||||||
vv_cache_write('watchdog', vv_wd_all());
|
vv_cache_write('watchdog', vv_wd_all());
|
||||||
|
|||||||
@@ -68,3 +68,13 @@
|
|||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
php "$SCRIPT_DIR/api_cache_writer.php"
|
php "$SCRIPT_DIR/api_cache_writer.php"
|
||||||
|
|
||||||
|
# Mesh traffic sample. Rides this job because it needs a steady once-a-minute cadence and adding
|
||||||
|
# a second per-minute cron for one append is more moving parts than the measurement is worth.
|
||||||
|
# Failure is ignored on purpose: a missed sample costs resolution in one window, and this job's
|
||||||
|
# actual purpose is the WebGUI cache.
|
||||||
|
php "$SCRIPT_DIR/mesh_traffic_sample.php" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
# Retry any mesh chat that could not be delivered when it was sent — a partner being asleep is
|
||||||
|
# the normal case for the message "my server is going down".
|
||||||
|
php "$SCRIPT_DIR/node_chat_receive.php" --flush >/dev/null 2>&1 || true
|
||||||
|
|||||||
@@ -28,6 +28,38 @@
|
|||||||
# finding for every hostname on the machine.
|
# finding for every hostname on the machine.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# It answers the questions nobody was going to ask.
|
||||||
|
# Both checks were already possible from the Auth tab, and both required somebody to open it
|
||||||
|
# and press a button on the right row. That is why one host here returned nothing but 5xx for
|
||||||
|
# months. A check that depends on being run is a check that is not running.
|
||||||
|
#
|
||||||
|
# Files findings; never applies remedies.
|
||||||
|
# The fixes are "start a container", "edit a rule", "change a default policy". Each of those
|
||||||
|
# is a judgement about intent, and a sweep that guessed at intent would be repairing a
|
||||||
|
# deliberate configuration as often as a broken one.
|
||||||
|
#
|
||||||
|
# Down is measured in time, not in samples.
|
||||||
|
# A host must have been failing for longer than AUTH_SWEEP_DOWN_MIN before anything is filed.
|
||||||
|
# Counting failed samples instead would file a finding for every hostname on the machine every
|
||||||
|
# time it reboots, and a report that cries wolf after every restart stops being read.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Read-only against the auth stack. Nothing here starts a container, edits a proxy host, changes
|
||||||
|
# a rule or alters a default policy — the two questions are answered by observation alone.
|
||||||
|
#
|
||||||
|
# --dry-run reports what it would file and writes no finding. --report only reads what is
|
||||||
|
# already filed, and is silent when clean, so it cannot manufacture noise for the Sunday report.
|
||||||
|
#
|
||||||
|
# Findings are filed, not notified. A sweep that raised an alert per host would be its own
|
||||||
|
# incident every time the array restarted.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -25,6 +25,40 @@
|
|||||||
# own created_on, which is a real date; nothing else is back-filled.
|
# own created_on, which is a real date; nothing else is back-filled.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# History is accumulated, never reconstructed.
|
||||||
|
# NPM holds only the present, so every past state this file knows about is one it observed at
|
||||||
|
# the time. Counts start at zero on first run and nothing is back-filled — first_seen is the
|
||||||
|
# single exception, seeded from NPM's own created_on because that is a real recorded date
|
||||||
|
# rather than an inference.
|
||||||
|
#
|
||||||
|
# A domain is retired on strikes, not on a single bad pass.
|
||||||
|
# A pass fails for a domain when it is absent from NPM's list or its expiry is already in the
|
||||||
|
# past — and NPM's list can come back short for reasons that have nothing to do with the
|
||||||
|
# certificate, such as an API hiccup or a restart mid-pass. CERT_HISTORY_STRIKES consecutive
|
||||||
|
# failures are required before a domain is retired, and it stays in the store afterwards, so
|
||||||
|
# one bad read can neither erase months of history nor hide a genuine expiry.
|
||||||
|
#
|
||||||
|
# The wrapper holds no logic.
|
||||||
|
# The work sits next to the NPM client it needs, because token handling lives in
|
||||||
|
# include/auth.php and a bash reimplementation would be a second copy of the thing most worth
|
||||||
|
# having only one of. Flags are forwarded verbatim.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# The store is replaced atomically: temp file, verified, then renamed over the original. A pass
|
||||||
|
# that fails partway leaves the previous history intact rather than a truncated file — this is
|
||||||
|
# the only record of what these certificates did, and there is no second copy to restore from.
|
||||||
|
#
|
||||||
|
# --dry-run reports every change it would make and writes nothing. --status only reads.
|
||||||
|
#
|
||||||
|
# Read-only against NPM. Certificates are observed; nothing here renews, deletes or edits one.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -40,6 +40,21 @@
|
|||||||
// downstream of something else here, and listing it alongside its own cause invites fixing the
|
// downstream of something else here, and listing it alongside its own cause invites fixing the
|
||||||
// symptom.
|
// symptom.
|
||||||
//
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Read-only. Opens certbot's logs and nothing else — no certificate is requested, renewed or
|
||||||
|
// deleted here, and no log is rotated or truncated.
|
||||||
|
//
|
||||||
|
// Bounded by file count and by bytes per file, so a directory that has grown to 639 MB across
|
||||||
|
// 1001 files cannot turn a page load into an unbounded read. Only the tail of each log is
|
||||||
|
// examined, because a run explains its failure at the end rather than the beginning.
|
||||||
|
//
|
||||||
|
// An unrecognised failure is reported as unclassified, never folded into the nearest category.
|
||||||
|
// A tidy chart that is wrong sends the operator to fix a domain that was never broken.
|
||||||
|
//
|
||||||
|
// The log directory is discovered from the NPM container rather than assumed, so a container
|
||||||
|
// path change surfaces as "no logs found" instead of an empty triage that reads as "no
|
||||||
|
// failures".
|
||||||
|
//
|
||||||
// RUNTIME MODES
|
// RUNTIME MODES
|
||||||
// cert_triage.php summary — categories, affected domains, and the causal reading
|
// cert_triage.php summary — categories, affected domains, and the causal reading
|
||||||
// cert_triage.php --json the same as JSON, for the Certs tab
|
// cert_triage.php --json the same as JSON, for the Certs tab
|
||||||
|
|||||||
@@ -30,6 +30,42 @@
|
|||||||
# fails renewals for domains that have nothing wrong with them.
|
# fails renewals for domains that have nothing wrong with them.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# One invocation is one data point.
|
||||||
|
# A single failure writes its reason three times — the ACME response, the traceback, and
|
||||||
|
# certbot's summary. Counting lines would report it as three and inflate whichever category
|
||||||
|
# happens to be the most verbose, which is the opposite of what triage is for.
|
||||||
|
#
|
||||||
|
# Causes are separated from consequences.
|
||||||
|
# Rate limiting is nearly always downstream: retries against a hostname with no DNS record
|
||||||
|
# exhaust the allowance, which then fails renewals for domains that have nothing wrong with
|
||||||
|
# them. Reporting the rate limit as the problem sends the operator to fix the wrong domains.
|
||||||
|
#
|
||||||
|
# Rotation suffix is the clock, not mtime.
|
||||||
|
# Every file in this directory carries the same mtime because they arrive as a synced set, so
|
||||||
|
# ordering by mtime returns an arbitrary answer that looks authoritative.
|
||||||
|
#
|
||||||
|
# Bounded by design, not by hope.
|
||||||
|
# 639 MB across 1001 files cannot be read in a page load. Only CERT_TRIAGE_FILES logs are
|
||||||
|
# opened, and only CERT_TRIAGE_MAX_BYTES from the end of each, because a failure explains
|
||||||
|
# itself at the end of the run rather than the start.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Read-only throughout. It opens certbot's logs and nothing else — no certificate is requested,
|
||||||
|
# renewed or deleted, and no log is rotated or truncated by anything here.
|
||||||
|
#
|
||||||
|
# Diagnosis only. This names why renewals failed; acting on that is the operator's, and
|
||||||
|
# cert_history.sh remains the thing that records what happened.
|
||||||
|
#
|
||||||
|
# The log directory is normally discovered from the NPM container rather than hardcoded, so a
|
||||||
|
# container path change surfaces as "no logs found" rather than as a silently empty triage.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -13,16 +13,39 @@
|
|||||||
// php Tools/conf_widget_check.php assertions, then the live summary
|
// php Tools/conf_widget_check.php assertions, then the live summary
|
||||||
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
||||||
//
|
//
|
||||||
// WHY IT ASSERTS AGAINST SNIPPETS AND NOT THE LIVE CONF
|
// DESIGN PRINCIPLES
|
||||||
// The live conf is the thing being described, so it cannot also be the thing that proves the
|
// Assertions run against snippets, never against the live conf.
|
||||||
// description right — an inference rule that silently stopped matching would keep passing as
|
// The live conf is the thing being described, so it cannot also be the thing that proves
|
||||||
// the conf drifted to suit it. The snippets are frozen copies of each convention as written,
|
// the description right — an inference rule that silently stopped matching would keep
|
||||||
// so a rule change that breaks one shows up here rather than as a wrong control on a page.
|
// passing as the conf drifted to suit it. The snippets are frozen copies of each
|
||||||
|
// convention as written, so a rule change that breaks one shows up here rather than as a
|
||||||
|
// wrong control on a page.
|
||||||
//
|
//
|
||||||
// WHAT AN INFERENCE IS NOT
|
// An inference is a drawing decision, not a validation.
|
||||||
// Consistent with confform.php, none of this validates. A number field carrying min and max is
|
// Consistent with confform.php, none of this validates. A number field carrying min and
|
||||||
// a courtesy to whoever is typing, not a promise the value is sensible — the consuming script
|
// max is a courtesy to whoever is typing, not a promise the value is sensible — the
|
||||||
// still owns that question.
|
// consuming script still owns that question.
|
||||||
|
//
|
||||||
|
// The live pass reports, it does not assert.
|
||||||
|
// What this host's master.conf infers to is a description of that file, not a verdict on
|
||||||
|
// it. Turning the live summary into pass/fail would make an unusual but legitimate
|
||||||
|
// setting look like a defect.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Read-only. Parses conf and reports; writes no conf, no store and no page.
|
||||||
|
//
|
||||||
|
// Exits non-zero only when a snippet assertion fails, so it can gate a commit without a real
|
||||||
|
// conf's contents ever being able to break the build.
|
||||||
|
//
|
||||||
|
// Never renders. It reports which control would be drawn; the drawing stays in confform.php,
|
||||||
|
// so this cannot disagree with the page by construction.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// php Tools/conf_widget_check.php assertions, then the live summary
|
||||||
|
// php Tools/conf_widget_check.php --list every live field and its inferred control
|
||||||
|
//
|
||||||
|
// Hand-run. Not scheduled and in no orchestrator — run it after touching _vv_conf_widget(),
|
||||||
|
// after adding a conf convention, or when a setting draws as the wrong control.
|
||||||
//
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/confform.php _vv_conf_parse_field_range(), vv_conf_key_is_secret()
|
// include/confform.php _vv_conf_parse_field_range(), vv_conf_key_is_secret()
|
||||||
|
|||||||
Executable
+217
@@ -0,0 +1,217 @@
|
|||||||
|
#!/usr/bin/php -q
|
||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Creates or updates the per-partner folder in folder.view3 — "<PartnerShort>-Fallback" — that
|
||||||
|
// holds the containers this host runs on the partner's behalf. Named from master.conf, so the
|
||||||
|
// folder name follows the hostnames rather than a value someone has to keep in step by hand.
|
||||||
|
//
|
||||||
|
// Onboard calls this on the mirror after deploying the owner's stack, so those containers land
|
||||||
|
// somewhere that says whose they are instead of scattering into the mirror's own groups.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Create-or-update, never replace. The named folder is found in folder.view3 if it exists and
|
||||||
|
// is amended in place; only its absence causes one to be created. Every other folder in the
|
||||||
|
// file, and every container already filed elsewhere, is left exactly as it was — this is one
|
||||||
|
// shelf in someone else's cupboard.
|
||||||
|
//
|
||||||
|
// Called during onboard on the mirror, after the owner's stack has been deployed there, so the
|
||||||
|
// containers exist by the time anything tries to file them.
|
||||||
|
//
|
||||||
|
// The icon is resolved separately from the folder and never gates it. --icon-only performs
|
||||||
|
// just that lookup and prints the URL, which is how onboard asks for it without writing
|
||||||
|
// anything.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// The name comes from master.conf, never from a second source.
|
||||||
|
// HOST2="unRAID-Jayred36" → "Jayred36-Fallback". The convention already existed by hand as
|
||||||
|
// "Jayred365-Fallback", which did not match any derivable value — so the folder is renamed
|
||||||
|
// to follow the conf rather than a conf field being invented to follow the folder.
|
||||||
|
//
|
||||||
|
// Upsert by name, never append.
|
||||||
|
// folder.view3 keys folders by a random id, so writing without looking produces a second
|
||||||
|
// folder with the same name and half the contents. That is exactly what happened here: a
|
||||||
|
// "Jayred365-Fallback" with the icon and no containers, beside one with the containers and
|
||||||
|
// no icon. Match on name, keep the existing id.
|
||||||
|
//
|
||||||
|
// This writes one folder, it does not replace the file.
|
||||||
|
// include/docker.php used to mirror Varaverk's whole folder store over the top of
|
||||||
|
// folder.view3's, destroying anything created in that plugin's UI. That write is gone. A
|
||||||
|
// co-writer that edits a single key it owns is a different thing from one that overwrites
|
||||||
|
// everything, and only the second kind loses data.
|
||||||
|
//
|
||||||
|
// Closest Emby user wins the icon, and a wrong guess costs nothing.
|
||||||
|
// The folder image is decoration. Exact match first, then lowest Levenshtein distance
|
||||||
|
// within a bound. "Jayred36" resolves to "Jayred365" (distance 1) over "Jayred" (2).
|
||||||
|
// This is deliberately unlike resolve_tailscale_ip's exact-prefix-plus-ambiguity-guard —
|
||||||
|
// there, a wrong match sends data to the wrong machine; here it picks the wrong avatar.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Atomic write — .vv.tmp then rename(), so folder.view3 never reads a truncated file.
|
||||||
|
// Absent plugin is a clean skip, not an error — nothing to do if folder.view3 is not installed.
|
||||||
|
// --dry-run prints the resulting folder and writes nothing.
|
||||||
|
// Icon resolution failing never blocks the folder: no image is a cosmetic loss, no folder is not.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// fallback_folder.php --host=HOST2 [--containers=A,B,C] [--icon=URL] [--dry-run]
|
||||||
|
// fallback_folder.php --host=HOST2 --icon-only resolve and print the icon URL, write nothing
|
||||||
|
//
|
||||||
|
// CONFIGURATION
|
||||||
|
// HOST1/HOST2… master.conf — the hostname the folder is named after
|
||||||
|
// HOST*_EMBY_URL where to look users up (local is fine, it is a server-side call)
|
||||||
|
// HOST*_EMBY_API_KEY "
|
||||||
|
// HOST*_EMBY_PUBLIC_URL base the ICON is built from. Must be reachable from a browser on
|
||||||
|
// either host, so localhost:8096 is not it — the icon renders in the
|
||||||
|
// WebGUI of whichever machine is looking. Empty means no icon.
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
$pluginDir = dirname(__DIR__);
|
||||||
|
require_once $pluginDir . '/include/config.php';
|
||||||
|
|
||||||
|
define('FV3_JSON', '/boot/config/plugins/folder.view3/docker.json');
|
||||||
|
|
||||||
|
// ── Args ──────────────────────────────────────────────────────────────────────
|
||||||
|
$opts = ['host' => '', 'containers' => '', 'icon' => '', 'dry-run' => false, 'icon-only' => false];
|
||||||
|
foreach (array_slice($argv, 1) as $a) {
|
||||||
|
if ($a === '--dry-run') { $opts['dry-run'] = true; continue; }
|
||||||
|
if ($a === '--icon-only'){ $opts['icon-only'] = true; continue; }
|
||||||
|
if (preg_match('/^--([a-z-]+)=(.*)$/', $a, $m)) $opts[$m[1]] = $m[2];
|
||||||
|
}
|
||||||
|
$hostId = strtoupper(trim($opts['host']));
|
||||||
|
if (!preg_match('/^HOST\d+$/', $hostId)) {
|
||||||
|
fwrite(STDERR, "usage: fallback_folder.php --host=HOST2 [--containers=A,B] [--dry-run]\n");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
$vars = vv_conf_vars();
|
||||||
|
$hostname = trim($vars[$hostId] ?? '');
|
||||||
|
if ($hostname === '') { fwrite(STDERR, "$hostId is not set in master.conf\n"); exit(1); }
|
||||||
|
|
||||||
|
|
||||||
|
// derive_short_name() in common.sh: lowercase, strip a leading "unraid-", capitalise.
|
||||||
|
$short = preg_replace('/^unraid-/i', '', $hostname);
|
||||||
|
$short = ucfirst($short);
|
||||||
|
$folderName = $short . '-Fallback';
|
||||||
|
|
||||||
|
// ── Icon: closest Emby user ───────────────────────────────────────────────────
|
||||||
|
function vv_closest_emby_icon(string $short, array $vars, string $meId): array {
|
||||||
|
$url = rtrim(trim($vars[$meId . '_EMBY_URL'] ?? ''), '/');
|
||||||
|
$key = trim($vars[$meId . '_EMBY_API_KEY'] ?? '');
|
||||||
|
$pub = rtrim(trim($vars[$meId . '_EMBY_PUBLIC_URL'] ?? ''), '/');
|
||||||
|
if ($url === '' || $key === '') return ['', 'no Emby url/key configured'];
|
||||||
|
|
||||||
|
$raw = @file_get_contents("$url/Users?api_key=" . urlencode($key), false,
|
||||||
|
stream_context_create(['http' => ['timeout' => 10]]));
|
||||||
|
$users = json_decode((string)$raw, true);
|
||||||
|
if (!is_array($users) || !$users) return ['', 'Emby returned no users'];
|
||||||
|
|
||||||
|
$needle = strtolower($short);
|
||||||
|
$best = null; $bestD = PHP_INT_MAX; $runnerUp = null;
|
||||||
|
foreach ($users as $u) {
|
||||||
|
$name = (string)($u['Name'] ?? '');
|
||||||
|
if ($name === '' || empty($u['Id'])) continue;
|
||||||
|
$d = levenshtein($needle, strtolower($name));
|
||||||
|
if ($d < $bestD) { $runnerUp = $best; $bestD = $d; $best = $u; }
|
||||||
|
elseif ($runnerUp === null || $d < levenshtein($needle, strtolower((string)$runnerUp['Name']))) {
|
||||||
|
$runnerUp = $u;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!$best) return ['', 'no candidate users'];
|
||||||
|
|
||||||
|
// Bound it so an unrelated name never wins by being the least-bad of a bad field.
|
||||||
|
$limit = max(2, (int)floor(strlen($needle) * 0.5));
|
||||||
|
if ($bestD > $limit) {
|
||||||
|
return ['', sprintf('closest was "%s" (distance %d > limit %d) — too far, no icon',
|
||||||
|
$best['Name'], $bestD, $limit)];
|
||||||
|
}
|
||||||
|
if ($pub === '') {
|
||||||
|
return ['', sprintf('matched Emby user "%s" (distance %d) but %s_EMBY_PUBLIC_URL is unset — '
|
||||||
|
. 'a localhost icon would not render in a browser', $best['Name'], $bestD, $meId)];
|
||||||
|
}
|
||||||
|
$tag = (string)($best['PrimaryImageTag'] ?? '');
|
||||||
|
if ($tag === '') return ['', sprintf('Emby user "%s" has no primary image', $best['Name'])];
|
||||||
|
|
||||||
|
$icon = sprintf('%s/Users/%s/Images/Primary?maxWidth=200&tag=%s&quality=90',
|
||||||
|
$pub, $best['Id'], $tag);
|
||||||
|
$note = sprintf('matched Emby user "%s" (distance %d%s)', $best['Name'], $bestD,
|
||||||
|
$runnerUp ? sprintf(', next "%s" at %d', $runnerUp['Name'],
|
||||||
|
levenshtein($needle, strtolower((string)$runnerUp['Name']))) : '');
|
||||||
|
return [$icon, $note];
|
||||||
|
}
|
||||||
|
|
||||||
|
$meId = strtoupper(vv_detect_host());
|
||||||
|
|
||||||
|
// --icon= skips the lookup entirely. The mirror needs this: it is being given a folder named
|
||||||
|
// after the OWNER, and the avatar lives in the owner's Emby — which the mirror has no key for and
|
||||||
|
// may not run at all. So the owner resolves the URL with --icon-only and hands it over, rather
|
||||||
|
// than the mirror guessing from a userbase it cannot see.
|
||||||
|
if (trim($opts['icon']) !== '') {
|
||||||
|
$icon = trim($opts['icon']);
|
||||||
|
$iconNote = 'supplied by caller';
|
||||||
|
} else {
|
||||||
|
[$icon, $iconNote] = vv_closest_emby_icon($short, $vars, $meId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// --icon-only: resolve and print, touch nothing. Exits non-zero when there is no icon, so a
|
||||||
|
// caller can tell "no image" from "empty string because something broke".
|
||||||
|
if ($opts['icon-only']) {
|
||||||
|
if ($icon === '') { fwrite(STDERR, "no icon: $iconNote\n"); exit(1); }
|
||||||
|
echo $icon . "\n";
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Upsert the folder ─────────────────────────────────────────────────────────
|
||||||
|
// A host does not keep a fallback folder for itself — the folder means "containers I run on
|
||||||
|
// SOMEONE ELSE's behalf", so naming it after this machine is always wrong. Checked here rather
|
||||||
|
// than at argument parsing, because --icon-only legitimately asks for THIS host's own avatar:
|
||||||
|
// the owner resolves its own picture to hand to the mirror, which is the whole point of that mode.
|
||||||
|
//
|
||||||
|
// Refused rather than created, because the failure is otherwise silent — an empty folder named
|
||||||
|
// after yourself looks plausible enough to survive a glance. One appeared on HOST1 exactly this
|
||||||
|
// way, when an older copy of this script ignored an unrecognised flag and ran the upsert anyway.
|
||||||
|
if (strcasecmp($hostId, strtoupper(vv_detect_host())) === 0) {
|
||||||
|
fwrite(STDERR, "$hostId is this host — a fallback folder is named after the PARTNER, not self\n");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!file_exists(FV3_JSON)) {
|
||||||
|
echo "folder.view3 is not installed — nothing to do\n";
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
$j = json_decode((string)@file_get_contents(FV3_JSON), true);
|
||||||
|
if (!is_array($j)) { fwrite(STDERR, "folder.view3 docker.json is unreadable\n"); exit(1); }
|
||||||
|
|
||||||
|
$containers = array_values(array_filter(array_map('trim', explode(',', (string)$opts['containers']))));
|
||||||
|
|
||||||
|
// Match on name, case-insensitively, so a hand-made folder is adopted rather than duplicated.
|
||||||
|
$targetId = null;
|
||||||
|
foreach ($j as $id => $f) {
|
||||||
|
if (is_array($f) && strcasecmp((string)($f['name'] ?? ''), $folderName) === 0) { $targetId = $id; break; }
|
||||||
|
}
|
||||||
|
$created = false;
|
||||||
|
if ($targetId === null) {
|
||||||
|
// folder.view3's own id shape: 20 chars of url-safe base64.
|
||||||
|
$targetId = substr(str_replace(['+', '/', '='], '', base64_encode(random_bytes(15))), 0, 20);
|
||||||
|
$j[$targetId] = ['name' => $folderName, 'icon' => '', 'settings' => ['', '', '1', '', '1', ''],
|
||||||
|
'regex' => '', 'containers' => [], 'containerImages' => []];
|
||||||
|
$created = true;
|
||||||
|
}
|
||||||
|
$j[$targetId]['name'] = $folderName;
|
||||||
|
if ($icon !== '') $j[$targetId]['icon'] = $icon;
|
||||||
|
foreach ($containers as $c) {
|
||||||
|
if (!in_array($c, (array)$j[$targetId]['containers'], true)) $j[$targetId]['containers'][] = $c;
|
||||||
|
}
|
||||||
|
$j[$targetId]['containers'] = array_values((array)$j[$targetId]['containers']);
|
||||||
|
|
||||||
|
printf("%s %s (id %s)\n", $created ? 'create' : 'update', $folderName, $targetId);
|
||||||
|
printf(" icon : %s\n", $icon !== '' ? $icon : '(none) — ' . $iconNote);
|
||||||
|
if ($icon !== '') printf(" via : %s\n", $iconNote);
|
||||||
|
printf(" containers: %s\n", implode(', ', (array)$j[$targetId]['containers']) ?: '(none)');
|
||||||
|
|
||||||
|
if ($opts['dry-run']) { echo " DRY RUN — nothing written\n"; exit(0); }
|
||||||
|
|
||||||
|
$tmp = FV3_JSON . '.vv.tmp';
|
||||||
|
if (file_put_contents($tmp, json_encode($j, JSON_UNESCAPED_SLASHES)) === false || !rename($tmp, FV3_JSON)) {
|
||||||
|
fwrite(STDERR, "failed to write " . FV3_JSON . "\n"); exit(1);
|
||||||
|
}
|
||||||
|
echo " written ✅\n";
|
||||||
Executable
+337
@@ -0,0 +1,337 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ==============================================================================================
|
||||||
|
# ============================= WebGUI JavaScript Checker ======================================
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# PURPOSE
|
||||||
|
# ==============================================================================================
|
||||||
|
# Finds the two JavaScript faults this plugin has actually shipped, neither of which any syntax
|
||||||
|
# check can see, because both are runtime behaviour:
|
||||||
|
#
|
||||||
|
# 1. An identifier declared inside one function and referenced from another. Throws
|
||||||
|
# ReferenceError on every call, kills the rest of the render, and passes php -l and
|
||||||
|
# node --check without complaint.
|
||||||
|
#
|
||||||
|
# 2. A fetch chain ending in an empty catch. Not error handling — error deletion. The request
|
||||||
|
# fails, nothing renders, nothing is logged, and the surface sits on "Loading…" forever.
|
||||||
|
#
|
||||||
|
# The two compound: on 2026-08-20 a cross-scope ReferenceError in the mesh chat was swallowed by
|
||||||
|
# an empty catch on every render. It presented as "the chat takes a minute to load" — the minute
|
||||||
|
# was the poller's backoff — and hours went into profiling PHP that was never slow. Once a catch
|
||||||
|
# reported the error, the fault named itself in one line.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# 1. Extract the JS from each <script> block in a PHP/page file
|
||||||
|
# 2. Strip PHP tags, comments, string literals, template-literal text and object keys
|
||||||
|
# 3. Cross-scope pass — declarations per function, then uses judged against them
|
||||||
|
# 4. Catch pass — an empty .catch() within 25 lines of a fetch()
|
||||||
|
#
|
||||||
|
# Reads only. Prints findings and exits non-zero when any are found, so it can gate a commit.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# A Checker That Cannot Fail Loudly Is Worse Than No Checker
|
||||||
|
# --self-test runs both detectors against a fixture carrying both faults and asserts they are
|
||||||
|
# found. It is not decoration. The first version of this scan reported a confident zero across
|
||||||
|
# the whole plugin because a regex was silently broken; the fixture is what catches that.
|
||||||
|
#
|
||||||
|
# Preprocessing Is Where The False Positives Die
|
||||||
|
# A raw scan produced ~1600 candidates, nearly all of them HTML attribute names, CSS keywords
|
||||||
|
# and English prose living inside template literals. Keeping only the ${...} expressions cut
|
||||||
|
# that to double digits. What remains after preprocessing is worth a human's attention.
|
||||||
|
#
|
||||||
|
# Report, Never Rewrite
|
||||||
|
# This prints file and line. It does not edit. A tool that silently "fixes" a false positive in
|
||||||
|
# a render path is a worse outcome than the fault it was hunting.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Escaped $ In Every Perl Character Class
|
||||||
|
# [A-Za-z_$] makes Perl interpolate $] — its version variable — and silently mangles the
|
||||||
|
# pattern into something that matches nothing. That mistake produced a false clean result twice
|
||||||
|
# while this was being written. Every class here writes \$, and --self-test would catch it
|
||||||
|
# returning.
|
||||||
|
#
|
||||||
|
# Known Parser Gaps, Stated Rather Than Hidden
|
||||||
|
# This is regex and brace counting, not a JavaScript parser. It over-reports and never
|
||||||
|
# under-reports, which is the safe direction: a clean run is meaningful, a dirty one needs a
|
||||||
|
# human. Preprocessing took the raw count from ~1600 to single digits by handling template
|
||||||
|
# literals, string literals, object keys, regex literals, destructuring, nested-function
|
||||||
|
# parameters and multi-declarator const.
|
||||||
|
#
|
||||||
|
# BASELINE as of 2026-08-21: three known false positives on a healthy tree —
|
||||||
|
# setup.php `to`, `id` in vvRenderOnboardPanel()
|
||||||
|
# Varaverk.page `s` in vvRenderMirrorOnboard()
|
||||||
|
# Each was read and confirmed harmless. Findings beyond those three are new and worth opening.
|
||||||
|
# If that baseline ever reaches zero, delete this paragraph rather than letting it rot.
|
||||||
|
#
|
||||||
|
# Non-Zero Exit On Findings
|
||||||
|
# So it can sit in front of a commit. A checker whose output has to be noticed by a human is a
|
||||||
|
# checker that stops being run.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# RUNTIME MODES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# js_check.sh
|
||||||
|
# Scan every pages/*.php, include/*.php and Varaverk.page.
|
||||||
|
#
|
||||||
|
# js_check.sh --scope | --catches
|
||||||
|
# One pass only.
|
||||||
|
#
|
||||||
|
# js_check.sh --self-test
|
||||||
|
# Verify both detectors against the built-in fixture. Run this after editing this file.
|
||||||
|
#
|
||||||
|
# js_check.sh <file> [...]
|
||||||
|
# Scan named files instead of the whole plugin.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
PLUGIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
WORK="$(mktemp -d)"
|
||||||
|
trap 'rm -rf "$WORK"' EXIT
|
||||||
|
|
||||||
|
DO_SCOPE=true; DO_CATCH=true; SELF_TEST=false
|
||||||
|
FILES=()
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--scope) DO_CATCH=false ;;
|
||||||
|
--catches) DO_SCOPE=false ;;
|
||||||
|
--self-test) SELF_TEST=true ;;
|
||||||
|
-*) echo "Unknown option: $arg" >&2; exit 1 ;;
|
||||||
|
*) FILES+=("$arg") ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
# ── Preprocessor ──────────────────────────────────────────────────────────────────────────────
|
||||||
|
cat > "$WORK/prep.pl" <<'PREP'
|
||||||
|
# NOTE: every character class escapes $ as \$. Bare [A-Za-z_$] interpolates $] (Perl's version)
|
||||||
|
# and silently matches nothing — see OPERATIONAL SAFEGUARDS.
|
||||||
|
local $/; my $s = <>;
|
||||||
|
$s =~ s{<\?=.*?\?>}{0}gs;
|
||||||
|
$s =~ s{<\?php.*?\?>}{}gs;
|
||||||
|
$s =~ s{/\*.*?\*/}{}gs;
|
||||||
|
$s =~ s{//[^\n]*}{}g;
|
||||||
|
# Template literals carry HTML, CSS and prose. Only the ${...} expressions are code.
|
||||||
|
$s =~ s{`((?:[^`\\]|\\.)*)`}{ " " . join(" ", $1 =~ m{\$\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}}g) . " " }ges;
|
||||||
|
$s =~ s{'(?:[^'\\\n]|\\.)*'}{""}g;
|
||||||
|
$s =~ s{"(?:[^"\\\n]|\\.)*"}{""}g;
|
||||||
|
# Regex literals. Only where one can legally begin — after ( = , : return — so division is not
|
||||||
|
# mistaken for a pattern. Their contents and flags are not identifiers: /\s*os|windows/i was
|
||||||
|
# reporting s, os and windows as undeclared variables.
|
||||||
|
$s =~ s{(=>\s*|[=(,:\[;?!]\s*|[&|]{2}\s*|\breturn\s+)/(?:[^/\\\n]|\\.)+/[gimsuy]*}{$1 0}g;
|
||||||
|
# Object-literal keys are not references. Anchored to { or , so a ternary keeps its operand.
|
||||||
|
$s =~ s/([{,]\s*)[A-Za-z_\$][A-Za-z0-9_\$]*\s*:/$1 /g;
|
||||||
|
print $s;
|
||||||
|
PREP
|
||||||
|
|
||||||
|
# ── Cross-scope detector ──────────────────────────────────────────────────────────────────────
|
||||||
|
cat > "$WORK/scope.awk" <<'SCOPE'
|
||||||
|
function fname(l, m) { if (match(l, /function[ \t]+[A-Za-z_$][A-Za-z0-9_$]*/)) { m=substr(l,RSTART,RLENGTH); sub(/function[ \t]+/,"",m); return m } return "" }
|
||||||
|
BEGIN {
|
||||||
|
# Built-ins and browser globals. Belt and braces — a parser slip that registers one of these as
|
||||||
|
# a local would otherwise report it in every other function in the file, which is exactly how
|
||||||
|
# Math appeared fifteen times while this was being written.
|
||||||
|
split("Math JSON Object Array String Number Boolean Date RegExp Error Promise Set Map WeakMap " \
|
||||||
|
"Symbol BigInt Intl console document window navigator location history screen localStorage " \
|
||||||
|
"sessionStorage fetch setTimeout setInterval clearTimeout clearInterval requestAnimationFrame " \
|
||||||
|
"parseInt parseFloat isNaN isFinite encodeURIComponent decodeURIComponent URLSearchParams " \
|
||||||
|
"FormData Headers Request Response AbortController CustomEvent Event Node Element " \
|
||||||
|
"getComputedStyle matchMedia structuredClone queueMicrotask btoa atob crypto performance " \
|
||||||
|
"globalThis undefined NaN Infinity ResizeObserver MutationObserver Uint32Array swal", g, " ")
|
||||||
|
for (gi in g) glob[g[gi]]=1
|
||||||
|
}
|
||||||
|
{ L[NR]=$0 }
|
||||||
|
END {
|
||||||
|
depth=0; cur=""
|
||||||
|
for (n=1; n<=NR; n++) {
|
||||||
|
line=L[n]
|
||||||
|
if (depth==0) { f=fname(line); if (f!="") cur=f }
|
||||||
|
scope = (cur=="" ? "(module)" : cur)
|
||||||
|
# Parameters of ANY function on this line, nested helpers included. Attributing a nested
|
||||||
|
# helper's params to the enclosing function is an approximation, and the right one: it can
|
||||||
|
# only suppress a report, never invent one, and nested params were the largest false-positive
|
||||||
|
# class in this codebase.
|
||||||
|
if (match(line, /function[ \t]*[A-Za-z0-9_$]*[ \t]*\([^)]*\)/)) {
|
||||||
|
pp=substr(line,RSTART,RLENGTH); sub(/.*\(/,"",pp); sub(/\).*/,"",pp)
|
||||||
|
c=split(pp,pa,","); for (k=1;k<=c;k++) { gsub(/[ \t]/,"",pa[k]); sub(/=.*/,"",pa[k]); if (pa[k]!="") decl[scope,pa[k]]=n } }
|
||||||
|
# Every declarator in the statement, not just the first: `const s = a, n = b` declares both,
|
||||||
|
# and reading only `s` left `n` looking undeclared wherever it was used.
|
||||||
|
if (match(line, /(const|let|var)[ \t]+/)) {
|
||||||
|
rest=substr(line,RSTART+RLENGTH)
|
||||||
|
# Destructuring binds every name inside the brackets: `for (const [folder, items] of …)`
|
||||||
|
# and `const { a, b } = obj`. Handled before the comma split, which cannot read them.
|
||||||
|
if (match(rest, /^[ \t]*[\[{][^\]}]*[\]}]/)) {
|
||||||
|
db=substr(rest,RSTART,RLENGTH)
|
||||||
|
while (match(db, /[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||||
|
nm2=substr(db,RSTART,RLENGTH)
|
||||||
|
decl[scope,nm2]=n; if (scope=="(module)") mod[nm2]=n
|
||||||
|
db=substr(db,RSTART+RLENGTH) }
|
||||||
|
}
|
||||||
|
cc=split(rest,dl,",")
|
||||||
|
for (kk=1;kk<=cc;kk++) {
|
||||||
|
# The identifier must be followed by "=" or end the segment. Without that test a comma
|
||||||
|
# inside the initialiser — `const pct = Math.min(a, Math.round(b))` — makes the second
|
||||||
|
# argument look like a second declarator, and Math gets registered as a local.
|
||||||
|
if (match(dl[kk], /^[ \t]*[A-Za-z_$][A-Za-z0-9_$]*[ \t]*(=[^=]|=$|$)/)) {
|
||||||
|
nm=dl[kk]; sub(/^[ \t]*/,"",nm); sub(/[ \t]*=.*$/,"",nm); gsub(/[ \t]/,"",nm)
|
||||||
|
if (nm!="") { decl[scope,nm]=n; if (scope=="(module)") mod[nm]=n }
|
||||||
|
} else break # stop at the first non-declarator; the rest is an expression
|
||||||
|
}
|
||||||
|
}
|
||||||
|
tmp=line
|
||||||
|
while (match(tmp, /(const|let|var)[ \t]+[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||||
|
d=substr(tmp,RSTART,RLENGTH); sub(/(const|let|var)[ \t]+/,"",d)
|
||||||
|
decl[scope,d]=n; if (scope=="(module)") mod[d]=n
|
||||||
|
tmp=substr(tmp,RSTART+RLENGTH) }
|
||||||
|
tmp=line
|
||||||
|
while (match(tmp, /\([^)]*\)[ \t]*=>/)) {
|
||||||
|
d=substr(tmp,RSTART,RLENGTH); gsub(/[()=>\t]/," ",d); gsub(/[{}\[\]]/," ",d)
|
||||||
|
c2=split(d,ap,","); for (k2=1;k2<=c2;k2++) { gsub(/[ \t]/,"",ap[k2]); sub(/=.*/,"",ap[k2]); if (ap[k2]!="") decl[scope,ap[k2]]=n }
|
||||||
|
tmp=substr(tmp,RSTART+RLENGTH) }
|
||||||
|
tmp=line
|
||||||
|
while (match(tmp, /[A-Za-z_$][A-Za-z0-9_$]*[ \t]*=>/)) {
|
||||||
|
d=substr(tmp,RSTART,RLENGTH); gsub(/[ \t=>]/,"",d); if (d!="") decl[scope,d]=n
|
||||||
|
tmp=substr(tmp,RSTART+RLENGTH) }
|
||||||
|
if (match(line, /for[ \t]*\([ \t]*(const|let|var)[ \t]+[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||||
|
d=substr(line,RSTART,RLENGTH); sub(/.*[ \t]/,"",d); decl[scope,d]=n }
|
||||||
|
if (match(line, /catch[ \t]*\([ \t]*[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||||
|
d=substr(line,RSTART,RLENGTH); sub(/.*\([ \t]*/,"",d); decl[scope,d]=n }
|
||||||
|
if (fname(line)!="") isfn[fname(line)]=1
|
||||||
|
for (i=1;i<=length(line);i++) { c=substr(line,i,1); if (c=="{") depth++; else if (c=="}") { depth--; if (depth<=0) { depth=0; cur="" } } }
|
||||||
|
}
|
||||||
|
depth=0; cur=""
|
||||||
|
for (n=1; n<=NR; n++) {
|
||||||
|
line=L[n]
|
||||||
|
if (depth==0) { f=fname(line); if (f!="") cur=f }
|
||||||
|
scope = (cur=="" ? "(module)" : cur)
|
||||||
|
if (scope!="(module)") {
|
||||||
|
tmp=line
|
||||||
|
while (match(tmp, /[A-Za-z_$][A-Za-z0-9_$]*/)) {
|
||||||
|
id=substr(tmp,RSTART,RLENGTH); pre=substr(tmp,RSTART-1,1)
|
||||||
|
tmp=substr(tmp,RSTART+RLENGTH)
|
||||||
|
if (pre=="." || id in isfn || id in mod || id in glob) continue
|
||||||
|
if ((scope,id) in decl) continue
|
||||||
|
for (o in decl) { split(o,q,SUBSEP); if (q[2]==id && q[1]!=scope && q[1]!="(module)") {
|
||||||
|
key=id SUBSEP scope; if (!(rep[key]++)) printf " line %d: %s used in %s() — declared only inside %s()\n", n, id, scope, q[1]; break } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (i=1;i<=length(line);i++) { c=substr(line,i,1); if (c=="{") depth++; else if (c=="}") { depth--; if (depth<=0) { depth=0; cur="" } } }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SCOPE
|
||||||
|
|
||||||
|
scan_scope() {
|
||||||
|
local f="$1"
|
||||||
|
awk '/<script/{p=1;next} /<\/script>/{p=0;print ""} p' "$f" | perl "$WORK/prep.pl" > "$WORK/x.js" 2>/dev/null
|
||||||
|
awk -f "$WORK/scope.awk" "$WORK/x.js" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# An empty catch within 25 lines of a fetch. Line distance rather than real chain parsing: the
|
||||||
|
# chains here are short, and a bare `catch {}` far from any request is usually a deliberate
|
||||||
|
# localStorage or execCommand guard, which this must not report.
|
||||||
|
scan_catches() {
|
||||||
|
local f="$1"
|
||||||
|
awk '/fetch\(|XMLHttpRequest/ { inf=NR }
|
||||||
|
/\.catch\(\s*\(\s*[a-z_]*\s*\)\s*=>\s*\{\s*\}\s*\)/ {
|
||||||
|
if (inf && NR-inf <= 25) printf " line %d: fetch chain ends in an empty catch\n", NR }' "$f"
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Self-test ─────────────────────────────────────────────────────────────────────────────────
|
||||||
|
if [[ "$SELF_TEST" == true ]]; then
|
||||||
|
cat > "$WORK/fixture.php" <<'FIX'
|
||||||
|
<script>
|
||||||
|
function outer() {
|
||||||
|
const paletteMap = { a: '#fff' };
|
||||||
|
return paletteMap;
|
||||||
|
}
|
||||||
|
function styler(x) {
|
||||||
|
// reads a const that lives in outer() — ReferenceError at runtime
|
||||||
|
if (paletteMap[x]) return paletteMap[x];
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
function loader() {
|
||||||
|
fetch('/api/thing')
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => { render(d); })
|
||||||
|
.catch(() => {});
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
FIX
|
||||||
|
fails=0
|
||||||
|
echo "── self-test ────────────────────────────────────────────────"
|
||||||
|
if scan_scope "$WORK/fixture.php" | grep -q "paletteMap"; then
|
||||||
|
echo " cross-scope detector PASS"
|
||||||
|
else
|
||||||
|
echo " cross-scope detector FAIL — known fault not reported"; fails=1
|
||||||
|
fi
|
||||||
|
if scan_catches "$WORK/fixture.php" | grep -q "empty catch"; then
|
||||||
|
echo " empty-catch detector PASS"
|
||||||
|
else
|
||||||
|
echo " empty-catch detector FAIL — known fault not reported"; fails=1
|
||||||
|
fi
|
||||||
|
# A clean file must stay clean, or the detector is merely reporting everything.
|
||||||
|
cat > "$WORK/clean.php" <<'CLN'
|
||||||
|
<script>
|
||||||
|
const paletteMap = { a: '#fff' };
|
||||||
|
function styler(x) { return paletteMap[x] || ''; }
|
||||||
|
function loader() {
|
||||||
|
fetch('/api/thing').then(r => r.json()).then(d => { styler(d); })
|
||||||
|
.catch(e => report('thing', e));
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
CLN
|
||||||
|
if [[ -z "$(scan_scope "$WORK/clean.php")$(scan_catches "$WORK/clean.php")" ]]; then
|
||||||
|
echo " clean file stays clean PASS"
|
||||||
|
else
|
||||||
|
echo " clean file stays clean FAIL — false positive on correct code"; fails=1
|
||||||
|
fi
|
||||||
|
echo "─────────────────────────────────────────────────────────────"
|
||||||
|
[[ "$fails" -eq 0 ]] && echo "self-test OK" || echo "SELF-TEST FAILED — do not trust a clean scan"
|
||||||
|
exit "$fails"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Scan ──────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
if [[ "${#FILES[@]}" -eq 0 ]]; then
|
||||||
|
while IFS= read -r p; do FILES+=("$p"); done < <(
|
||||||
|
find "$PLUGIN_DIR/pages" "$PLUGIN_DIR/include" -maxdepth 1 -name '*.php' 2>/dev/null
|
||||||
|
[[ -f "$PLUGIN_DIR/Varaverk.page" ]] && echo "$PLUGIN_DIR/Varaverk.page"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
found=0
|
||||||
|
for f in "${FILES[@]}"; do
|
||||||
|
[[ -f "$f" ]] || continue
|
||||||
|
grep -q "<script" "$f" 2>/dev/null || continue
|
||||||
|
out=""
|
||||||
|
[[ "$DO_SCOPE" == true ]] && out+="$(scan_scope "$f")"
|
||||||
|
if [[ "$DO_CATCH" == true ]]; then
|
||||||
|
c="$(scan_catches "$f")"
|
||||||
|
[[ -n "$c" ]] && out+=$'\n'"$c"
|
||||||
|
fi
|
||||||
|
out="$(echo "$out" | sed '/^$/d')"
|
||||||
|
if [[ -n "$out" ]]; then
|
||||||
|
echo "### $(basename "$f")"
|
||||||
|
echo "$out"
|
||||||
|
found=$(( found + $(echo "$out" | grep -c '^ line') ))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
echo
|
||||||
|
if [[ "$found" -eq 0 ]]; then
|
||||||
|
echo "clean — no cross-scope references, no silent fetch catches"
|
||||||
|
else
|
||||||
|
echo "$found finding(s). Cross-scope hits over-report: regex literals, nested-function"
|
||||||
|
echo "parameters, destructured for-of and multi-declarator const each read as undeclared."
|
||||||
|
echo "Read each before changing anything."
|
||||||
|
fi
|
||||||
|
exit $(( found > 0 ? 1 : 0 ))
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Sample per-peer Tailscale byte counters, once a minute, so the Partnership page can report
|
||||||
|
// how much has actually moved between the servers in this mesh over a window.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Appends one line per mesh peer to data/db/mesh_traffic.db:
|
||||||
|
//
|
||||||
|
// epoch|peer-hostname|txbytes|rxbytes
|
||||||
|
//
|
||||||
|
// Called from Tools/api_cache_writer.sh, which already runs every minute. A window total is
|
||||||
|
// then the difference between the newest sample and the oldest one still inside that window.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// Measure the link, not the tool.
|
||||||
|
// The previous card totalled rsync's own logs, so it could only ever describe rsync — and
|
||||||
|
// said "no data moved" while SSH, the arr APIs, conf pushes and the Unraid API were all
|
||||||
|
// using the same link. Tailscale counts the bytes on the wire, whatever sent them.
|
||||||
|
//
|
||||||
|
// Mesh peers only.
|
||||||
|
// The tailnet holds phones and workstations. Filtered against the HOST* hostnames in
|
||||||
|
// master.conf so this measures the partnership, not the tailnet.
|
||||||
|
//
|
||||||
|
// Absolute counters are stored, never deltas.
|
||||||
|
// A delta computed at write time bakes in whatever the sampling interval happened to be
|
||||||
|
// and cannot be re-derived if a run is missed. Storing the raw counter means a gap costs
|
||||||
|
// resolution, not correctness.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Counter resets are the reader's problem, not this file's — tailscaled restarting returns
|
||||||
|
// the counters to zero, and a sampler that tried to compensate would have to guess when.
|
||||||
|
// Storing raw values leaves the evidence intact: a sample lower than the one before it is a
|
||||||
|
// restart, and it is unambiguous.
|
||||||
|
//
|
||||||
|
// Trimmed to VV_MESH_KEEP_DAYS on every run, so the file cannot grow without bound. At one
|
||||||
|
// sample per peer per minute that is ~1,440 lines/peer/day.
|
||||||
|
//
|
||||||
|
// Silent no-op when tailscale is absent or returns nothing parseable. This runs every minute
|
||||||
|
// from a cache writer; a mesh sampler is not worth a log line per minute when it has nothing
|
||||||
|
// to add.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// php mesh_traffic_sample.php append one sample per mesh peer
|
||||||
|
// php mesh_traffic_sample.php --show print what it would record, write nothing
|
||||||
|
//
|
||||||
|
// CONFIGURATION
|
||||||
|
// master.conf
|
||||||
|
// HOST1, HOST2, … the hostname of each node in the mesh. Every HOST<n> with a non-empty
|
||||||
|
// value becomes a name matched against tailscale's peer list; a peer that
|
||||||
|
// is not one of them is skipped, so a device that merely shares the
|
||||||
|
// tailnet is never recorded as mesh traffic. No HOST<n> set at all means
|
||||||
|
// there is no mesh to sample and the run exits without writing.
|
||||||
|
//
|
||||||
|
// Nothing is read from host*.conf. This samples the local tailscale daemon, so it needs to
|
||||||
|
// know which peers count and nothing about how to reach them.
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// tailscale status --json the counters
|
||||||
|
// include/config.php DATA_DIR, vv_conf_vars() for the HOST* list
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
require_once dirname(__DIR__) . '/include/config.php';
|
||||||
|
|
||||||
|
define('VV_MESH_DB', DATA_DIR . '/db/mesh_traffic.db');
|
||||||
|
define('VV_MESH_KEEP_DAYS', 40); // a 30-day window needs a sample older than 30 days
|
||||||
|
|
||||||
|
$show = in_array('--show', $argv ?? [], true);
|
||||||
|
|
||||||
|
$raw = shell_exec('tailscale status --json 2>/dev/null');
|
||||||
|
$js = json_decode((string)$raw, true);
|
||||||
|
if (!is_array($js) || empty($js['Peer'])) exit(0);
|
||||||
|
|
||||||
|
// The hostnames this mesh is made of. Anything else on the tailnet is somebody's laptop.
|
||||||
|
$vars = vv_conf_vars();
|
||||||
|
$mesh = [];
|
||||||
|
foreach ($vars as $k => $v) {
|
||||||
|
if (preg_match('/^HOST\d+$/', $k) && trim((string)$v) !== '') $mesh[strtolower(trim($v))] = true;
|
||||||
|
}
|
||||||
|
if (!$mesh) exit(0);
|
||||||
|
|
||||||
|
$now = time();
|
||||||
|
$lines = [];
|
||||||
|
foreach ($js['Peer'] as $peer) {
|
||||||
|
$host = strtolower(trim((string)($peer['HostName'] ?? '')));
|
||||||
|
if ($host === '') continue;
|
||||||
|
|
||||||
|
// Same unambiguous-prefix rule the rest of the partnership layer uses: the tailnet name and
|
||||||
|
// the conf hostname are different strings and nothing keeps them in step.
|
||||||
|
$match = null;
|
||||||
|
if (isset($mesh[$host])) {
|
||||||
|
$match = $host;
|
||||||
|
} else {
|
||||||
|
$cand = [];
|
||||||
|
foreach (array_keys($mesh) as $m) {
|
||||||
|
if (str_starts_with($host, $m) || str_starts_with($m, $host)) $cand[] = $m;
|
||||||
|
}
|
||||||
|
if (count($cand) === 1) $match = $cand[0];
|
||||||
|
}
|
||||||
|
if ($match === null) continue;
|
||||||
|
|
||||||
|
$tx = (int)($peer['TxBytes'] ?? 0);
|
||||||
|
$rx = (int)($peer['RxBytes'] ?? 0);
|
||||||
|
if ($tx === 0 && $rx === 0) continue;
|
||||||
|
$lines[] = $now . '|' . $match . '|' . $tx . '|' . $rx;
|
||||||
|
}
|
||||||
|
if (!$lines) exit(0);
|
||||||
|
|
||||||
|
if ($show) { echo implode("\n", $lines) . "\n"; exit(0); }
|
||||||
|
|
||||||
|
@mkdir(dirname(VV_MESH_DB), 0755, true);
|
||||||
|
@file_put_contents(VV_MESH_DB, implode("\n", $lines) . "\n", FILE_APPEND | LOCK_EX);
|
||||||
|
|
||||||
|
// Trim in place. Read-filter-rewrite rather than append-only truncation, because the cut is by
|
||||||
|
// age and the file is not ordered by peer.
|
||||||
|
$cutoff = $now - (VV_MESH_KEEP_DAYS * 86400);
|
||||||
|
$all = @file(VV_MESH_DB, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [];
|
||||||
|
if (count($all) > 200) {
|
||||||
|
$keep = [];
|
||||||
|
foreach ($all as $l) {
|
||||||
|
$ts = (int)strtok($l, '|');
|
||||||
|
if ($ts >= $cutoff) $keep[] = $l;
|
||||||
|
}
|
||||||
|
if (count($keep) !== count($all)) {
|
||||||
|
$tmp = VV_MESH_DB . '.tmp';
|
||||||
|
if (@file_put_contents($tmp, implode("\n", $keep) . "\n") !== false) @rename($tmp, VV_MESH_DB);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Reproduce the owner's docker folder layout on the mirror, for the containers the owner
|
||||||
|
// actually deployed there. Sonarr lands in "Arrs Stack", NginxProxyManager and Lldap in
|
||||||
|
// "Networking", the databases in "Databases" — the same shelves they sit on at home.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Two modes, one file, piped over SSH:
|
||||||
|
//
|
||||||
|
// --export --containers=a,b,c on the OWNER. Reads folder.view3's docker.json, intersects
|
||||||
|
// each folder with the deployed list, prints a JSON plan.
|
||||||
|
// --import on the MIRROR. Reads that plan on stdin and upserts each
|
||||||
|
// folder by name into BOTH the mirror's folder.view3 docker.json
|
||||||
|
// and Varaverk's own docker_folders.json.
|
||||||
|
//
|
||||||
|
// Onboard Step 12 runs the pair. Nothing is assumed about the mirror's layout: folders it
|
||||||
|
// already has are matched by name and extended, never duplicated or replaced.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// The owner's layout is the source, and only the deployed subset travels.
|
||||||
|
// "Arrs Stack" holds ten containers on the owner and five of them were deployed, so the
|
||||||
|
// mirror's "Arrs Stack" gets those five. Sending the whole list would name containers the
|
||||||
|
// mirror does not have, and folder.view3 renders those as empty tiles.
|
||||||
|
//
|
||||||
|
// One folder per owner folder — not one folder for everything.
|
||||||
|
// Step 12 used to put all thirteen containers into "<Owner>-Fallback". That says whose
|
||||||
|
// they are and nothing about what they do, and it is the wrong shape for a stack that runs
|
||||||
|
// continuously rather than only during a failover.
|
||||||
|
//
|
||||||
|
// The fallback folder is for what is genuinely fallback-only.
|
||||||
|
// Anything in PARTNERSHIP_FALLBACK_ONLY, plus anything deployed that the owner does not
|
||||||
|
// file anywhere, goes to "<OwnerShort>-Fallback". A container that runs on the mirror all
|
||||||
|
// the time belongs with its peers; a container that exists only to cover the owner going
|
||||||
|
// dark belongs in a folder named after the owner.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Upsert by name, never append — folder.view3 keys by random id, so a blind write produces a
|
||||||
|
// second folder with the same name and half the contents.
|
||||||
|
//
|
||||||
|
// Atomic write: .vv.tmp then rename(), so folder.view3 never reads a truncated file.
|
||||||
|
//
|
||||||
|
// Both stores or the layout is only half real. The Monitor container card reads Varaverk's
|
||||||
|
// docker_folders.json, never folder.view3's, and include/docker.php seeds it exactly once —
|
||||||
|
// when the file is absent. A mirror onboarded before this step existed had already been
|
||||||
|
// seeded, so writing only folder.view3 left the card showing the old single lump forever.
|
||||||
|
//
|
||||||
|
// Absent plugin is no longer a skip on import. folder.view3 is optional; Varaverk's own
|
||||||
|
// store is not, and it is the one the card reads. Export still needs the plugin and says so.
|
||||||
|
//
|
||||||
|
// Import trusts nothing about shape: every folder needs a non-empty name and an array of
|
||||||
|
// container names, and anything else in the payload is ignored rather than merged.
|
||||||
|
//
|
||||||
|
// Icons are carried across as URLs. They are the owner's own icon values, already resolvable
|
||||||
|
// from any host, and an icon that fails to load is cosmetic.
|
||||||
|
//
|
||||||
|
// REQUEST
|
||||||
|
// mirror_folders.php --export --containers=Sonarr,Radarr[,…] [--fallback-only=a,b] [--owner=HOST1]
|
||||||
|
// mirror_folders.php --import [--dry-run] (plan on stdin)
|
||||||
|
//
|
||||||
|
// RESPONSE
|
||||||
|
// export: JSON plan on stdout — {"folders":[{"name":…,"icon":…,"containers":[…]}, …],
|
||||||
|
// "unfiled":[…]}
|
||||||
|
// import: one line per folder written, then a count
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// mirror_folders.php --export
|
||||||
|
// Print this host's folder layout as JSON, for the owner to compute a plan from.
|
||||||
|
//
|
||||||
|
// mirror_folders.php --import
|
||||||
|
// Read a layout on stdin and apply it to this host's folder.view3.
|
||||||
|
//
|
||||||
|
// mirror_folders.php --containers=<csv>
|
||||||
|
// Restrict the operation to these container names.
|
||||||
|
//
|
||||||
|
// mirror_folders.php --fallback-only=<csv>
|
||||||
|
// Place only these into the partner's fallback folder, leaving the rest where they are.
|
||||||
|
//
|
||||||
|
// --dry-run
|
||||||
|
// Compute and report the plan without writing folder.view3. Combines with any of the above.
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// /boot/config/plugins/folder.view3/docker.json the third-party layout, optional on import
|
||||||
|
// SCRIPTS_DIR/docker_folders.json Varaverk's own layout — what the card reads
|
||||||
|
// Tools/fallback_folder.php still owns the "<Owner>-Fallback" folder
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
$pluginDir = dirname(__DIR__);
|
||||||
|
require_once $pluginDir . '/include/config.php';
|
||||||
|
|
||||||
|
define('FV3_JSON', '/boot/config/plugins/folder.view3/docker.json');
|
||||||
|
// Varaverk's own store. Written as well as folder.view3's, because the Monitor container card
|
||||||
|
// reads THIS file and nothing else — include/docker.php imports folder.view3 exactly once, when
|
||||||
|
// this file does not yet exist. On a mirror onboarded before the layout step existed, that
|
||||||
|
// bootstrap had already happened, so the card went on showing one lump "<Owner>-Fallback"
|
||||||
|
// holding the entire auth and arr stacks while folder.view3 next to it showed the real shelves.
|
||||||
|
define('VV_STORE', SCRIPTS_DIR . '/docker_folders.json');
|
||||||
|
|
||||||
|
$opts = getopt('', ['export', 'import', 'containers:', 'fallback-only:', 'dry-run']);
|
||||||
|
|
||||||
|
$listArg = function (string $k) use ($opts): array {
|
||||||
|
if (!isset($opts[$k])) return [];
|
||||||
|
return array_values(array_filter(array_map('trim', explode(',', (string)$opts[$k])), 'strlen'));
|
||||||
|
};
|
||||||
|
|
||||||
|
// ── Load ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
// folder.view3 is optional on the mirror. Export needs it and has nothing to say without it;
|
||||||
|
// import does not — Varaverk's own store is the one the card reads, and it is always writable.
|
||||||
|
$haveFv3 = file_exists(FV3_JSON);
|
||||||
|
if (!$haveFv3 && isset($opts['export'])) {
|
||||||
|
echo json_encode(['folders' => [], 'unfiled' => $listArg('containers')]) . "\n";
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
$fv3 = [];
|
||||||
|
if ($haveFv3) {
|
||||||
|
$fv3 = json_decode((string)@file_get_contents(FV3_JSON), true);
|
||||||
|
if (!is_array($fv3)) { fwrite(STDERR, "folder.view3 docker.json is unreadable\n"); exit(1); }
|
||||||
|
}
|
||||||
|
// ── Export ───────────────────────────────────────────────────────────────────────────────────
|
||||||
|
if (isset($opts['export'])) {
|
||||||
|
$deployed = $listArg('containers');
|
||||||
|
$fallbackOnly = array_map('strtolower', $listArg('fallback-only'));
|
||||||
|
if (!$deployed) { fwrite(STDERR, "--containers is required for --export\n"); exit(2); }
|
||||||
|
|
||||||
|
// Case-insensitive membership, because container names come from XML filenames on one side
|
||||||
|
// and docker on the other, and those have disagreed on capitalisation before (my-prowlarr).
|
||||||
|
$remaining = [];
|
||||||
|
foreach ($deployed as $c) $remaining[strtolower($c)] = $c;
|
||||||
|
|
||||||
|
$plan = [];
|
||||||
|
foreach ($fv3 as $f) {
|
||||||
|
if (!is_array($f)) continue;
|
||||||
|
$name = trim((string)($f['name'] ?? ''));
|
||||||
|
if ($name === '') continue;
|
||||||
|
// The owner's own fallback folders describe the owner's coverage of someone else. They
|
||||||
|
// are not part of the mirror's layout and copying them would be nonsense on that host.
|
||||||
|
if (preg_match('/-Fallback$/i', $name)) continue;
|
||||||
|
|
||||||
|
$members = [];
|
||||||
|
foreach ((array)($f['containers'] ?? []) as $c) {
|
||||||
|
$lc = strtolower(trim((string)$c));
|
||||||
|
if ($lc === '' || !isset($remaining[$lc])) continue;
|
||||||
|
if (in_array($lc, $fallbackOnly, true)) continue; // claimed by the fallback folder
|
||||||
|
$members[] = $remaining[$lc];
|
||||||
|
unset($remaining[$lc]);
|
||||||
|
}
|
||||||
|
if ($members) $plan[] = ['name' => $name, 'icon' => (string)($f['icon'] ?? ''), 'containers' => $members];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whatever the owner files nowhere, plus everything explicitly marked fallback-only.
|
||||||
|
echo json_encode(['folders' => $plan, 'unfiled' => array_values($remaining)], JSON_UNESCAPED_SLASHES) . "\n";
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ── Import ───────────────────────────────────────────────────────────────────────────────────
|
||||||
|
if (!isset($opts['import'])) {
|
||||||
|
fwrite(STDERR, "usage: mirror_folders.php --export --containers=… | --import\n");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
$raw = stream_get_contents(STDIN);
|
||||||
|
$plan = json_decode((string)$raw, true);
|
||||||
|
if (!is_array($plan) || !isset($plan['folders']) || !is_array($plan['folders'])) {
|
||||||
|
fwrite(STDERR, "import: no usable plan on stdin\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Applied to both stores, so they cannot drift apart the way they already did once. Takes a
|
||||||
|
// store, returns the store with the plan folded in plus what changed — no writing, no printing,
|
||||||
|
// because the two callers report differently.
|
||||||
|
$applyPlan = function (array $store) use ($plan): array {
|
||||||
|
$written = 0;
|
||||||
|
$lines = [];
|
||||||
|
foreach ($plan['folders'] as $spec) {
|
||||||
|
if (!is_array($spec)) continue;
|
||||||
|
$name = trim((string)($spec['name'] ?? ''));
|
||||||
|
$cs = array_values(array_filter(array_map('trim', (array)($spec['containers'] ?? [])), 'strlen'));
|
||||||
|
if ($name === '' || !$cs) continue;
|
||||||
|
|
||||||
|
$targetId = null;
|
||||||
|
foreach ($store as $id => $f) {
|
||||||
|
if (is_array($f) && strcasecmp((string)($f['name'] ?? ''), $name) === 0) { $targetId = $id; break; }
|
||||||
|
}
|
||||||
|
$created = false;
|
||||||
|
if ($targetId === null) {
|
||||||
|
// folder.view3's own id shape: 20 chars of url-safe base64.
|
||||||
|
$targetId = substr(str_replace(['+', '/', '='], '', base64_encode(random_bytes(15))), 0, 20);
|
||||||
|
$store[$targetId] = ['name' => $name, 'icon' => '', 'settings' => ['', '', '1', '', '1', ''],
|
||||||
|
'regex' => '', 'containers' => [], 'containerImages' => []];
|
||||||
|
$created = true;
|
||||||
|
}
|
||||||
|
$store[$targetId]['name'] = $name;
|
||||||
|
// Only fill an icon that is missing — a mirror that has styled its own folder keeps its choice.
|
||||||
|
if (($spec['icon'] ?? '') !== '' && trim((string)($store[$targetId]['icon'] ?? '')) === '') {
|
||||||
|
$store[$targetId]['icon'] = (string)$spec['icon'];
|
||||||
|
}
|
||||||
|
$have = (array)($store[$targetId]['containers'] ?? []);
|
||||||
|
$lc = array_map('strtolower', array_map('strval', $have));
|
||||||
|
$added = 0;
|
||||||
|
foreach ($cs as $c) {
|
||||||
|
if (in_array(strtolower($c), $lc, true)) continue;
|
||||||
|
$have[] = $c; $lc[] = strtolower($c); $added++;
|
||||||
|
}
|
||||||
|
$store[$targetId]['containers'] = array_values($have);
|
||||||
|
$lines[] = sprintf(" %-7s %-22s +%d (%s)", $created ? 'create' : 'update', $name, $added, implode(', ', $cs));
|
||||||
|
$written++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Take the filed containers back out of any "-Fallback" folder ─────────────────────────
|
||||||
|
// A container that now sits in "Arrs Stack" must not also sit in "Gmer4Lfe-Fallback". The
|
||||||
|
// fallback folder answers "what is this host covering for the owner", and an earlier Step 12
|
||||||
|
// dumped every deployed container into it — so it claimed the whole auth and arr stacks,
|
||||||
|
// which run here continuously and are not failover coverage at all.
|
||||||
|
//
|
||||||
|
// Only containers this plan just filed are removed. Anything the operator put in that folder
|
||||||
|
// by hand, or that the onboard filed there deliberately as fallback-only, is left alone.
|
||||||
|
$filed = [];
|
||||||
|
foreach ($plan['folders'] as $spec) {
|
||||||
|
foreach ((array)($spec['containers'] ?? []) as $c) $filed[strtolower(trim((string)$c))] = true;
|
||||||
|
}
|
||||||
|
$pruned = 0;
|
||||||
|
foreach ($store as $id => $f) {
|
||||||
|
if (!is_array($f) || !preg_match('/-Fallback$/i', (string)($f['name'] ?? ''))) continue;
|
||||||
|
$keep = [];
|
||||||
|
foreach ((array)($f['containers'] ?? []) as $c) {
|
||||||
|
if (isset($filed[strtolower(trim((string)$c))])) { $pruned++; continue; }
|
||||||
|
$keep[] = $c;
|
||||||
|
}
|
||||||
|
$store[$id]['containers'] = array_values($keep);
|
||||||
|
}
|
||||||
|
return [$store, $written, $pruned, $lines];
|
||||||
|
};
|
||||||
|
|
||||||
|
$dryRun = isset($opts['dry-run']);
|
||||||
|
|
||||||
|
// ── folder.view3's file ──────────────────────────────────────────────────────────────────────
|
||||||
|
$fvWritten = 0;
|
||||||
|
if ($haveFv3) {
|
||||||
|
[$fv3, $fvWritten, $fvPruned, $fvLines] = $applyPlan($fv3);
|
||||||
|
foreach ($fvLines as $l) echo $l . "\n";
|
||||||
|
if ($fvPruned) printf(" pruned %d container(s) from -Fallback folder(s) — they are filed properly now\n", $fvPruned);
|
||||||
|
if (($fvWritten || $fvPruned) && !$dryRun) {
|
||||||
|
$tmp = FV3_JSON . '.vv.tmp';
|
||||||
|
if (file_put_contents($tmp, json_encode($fv3, JSON_UNESCAPED_SLASHES)) === false || !rename($tmp, FV3_JSON)) {
|
||||||
|
fwrite(STDERR, "failed to write " . FV3_JSON . "\n"); exit(1);
|
||||||
|
}
|
||||||
|
printf(" %d folder(s) written to folder.view3 ✅\n", $fvWritten);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
echo " folder.view3 not installed — Varaverk's own layout only\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Varaverk's own store ─────────────────────────────────────────────────────────────────────
|
||||||
|
// Seeded from folder.view3 when it does not exist yet, which is the same bootstrap
|
||||||
|
// include/docker.php performs — done here too so the very first import lands on a real layout
|
||||||
|
// rather than an empty file.
|
||||||
|
$vvStore = [];
|
||||||
|
if (file_exists(VV_STORE)) {
|
||||||
|
$vvStore = json_decode((string)@file_get_contents(VV_STORE), true);
|
||||||
|
if (!is_array($vvStore)) $vvStore = [];
|
||||||
|
} elseif ($haveFv3) {
|
||||||
|
$vvStore = $fv3;
|
||||||
|
}
|
||||||
|
[$vvStore, $vvWritten, $vvPruned, ] = $applyPlan($vvStore);
|
||||||
|
if (($vvWritten || $vvPruned) && !$dryRun) {
|
||||||
|
@mkdir(dirname(VV_STORE), 0755, true);
|
||||||
|
$tmp = VV_STORE . '.vv.tmp';
|
||||||
|
if (file_put_contents($tmp, json_encode($vvStore, JSON_UNESCAPED_SLASHES)) === false || !rename($tmp, VV_STORE)) {
|
||||||
|
fwrite(STDERR, "failed to write " . VV_STORE . "\n"); exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printf(" %d folder(s) %s Varaverk's layout%s\n", $vvWritten,
|
||||||
|
$dryRun ? 'would be written to' : 'written to',
|
||||||
|
$vvPruned ? sprintf(" (%d unfiled from -Fallback)", $vvPruned) : '');
|
||||||
|
|
||||||
|
if ($dryRun) echo " DRY RUN — nothing written\n";
|
||||||
|
if (!$fvWritten && !$vvWritten) echo " nothing to write\n";
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Accept one mesh message from a partner and store it. Invoked over SSH by the sending node's
|
||||||
|
// vv_nc_deliver(), with the record on stdin.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Exit 0 means stored. The sender treats anything else as undelivered and spools for retry, so
|
||||||
|
// a non-zero exit here is a message that will arrive later rather than one that is lost.
|
||||||
|
//
|
||||||
|
// Two jobs in one file because they are two halves of the same contract: --channel takes what
|
||||||
|
// a partner is delivering now, --flush pushes what this host failed to deliver earlier. A node
|
||||||
|
// is both a receiver and a sender, and splitting them would mean two files that must agree on
|
||||||
|
// the same spool layout.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// The exit code is the receipt, and it is the only one.
|
||||||
|
// There is no acknowledgement message and no reply body. The sender is an SSH command that
|
||||||
|
// already has an exit status, so inventing a second channel to say the same thing would
|
||||||
|
// give the two ways to disagree.
|
||||||
|
//
|
||||||
|
// Refusing is cheaper than being wrong.
|
||||||
|
// An unparseable record, an unknown channel or a failed write all exit non-zero and leave
|
||||||
|
// the message in the sender's spool. A retry costs one SSH round trip; a message accepted
|
||||||
|
// and dropped is gone with nothing recording that it existed.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Reached only over SSH with a key this mesh installed, so the caller already has root. This
|
||||||
|
// file therefore validates shape, not authority — there is no privilege here to protect that
|
||||||
|
// the transport has not already granted.
|
||||||
|
//
|
||||||
|
// The channel is resolved against this host's own membership. A name that is not a channel
|
||||||
|
// this machine belongs to is refused, so the argument cannot address a path outside the store.
|
||||||
|
//
|
||||||
|
// Storage is append-and-trim through vv_nc_append(), which is idempotent on message id — a
|
||||||
|
// retry of something that already landed is a no-op rather than a duplicate.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// Not invoked by hand — the sending node's vv_nc_deliver() runs it over SSH.
|
||||||
|
//
|
||||||
|
// php node_chat_receive.php --channel=<id> store one JSON record, read from stdin
|
||||||
|
// php node_chat_receive.php --flush retry this host's own undelivered spool
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/node_chat.php vv_nc_append(), vv_nc_channel(), vv_nc_flush_spool()
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
require_once dirname(__DIR__) . '/include/node_chat.php';
|
||||||
|
|
||||||
|
$args = $argv ?? [];
|
||||||
|
|
||||||
|
if (in_array('--flush', $args, true)) {
|
||||||
|
$n = vv_nc_flush_spool();
|
||||||
|
if ($n) echo "delivered $n queued message(s)\n";
|
||||||
|
exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
$channel = '';
|
||||||
|
foreach ($args as $a) {
|
||||||
|
if (str_starts_with($a, '--channel=')) $channel = substr($a, 10);
|
||||||
|
}
|
||||||
|
if ($channel === '' || !vv_nc_channel($channel)) {
|
||||||
|
fwrite(STDERR, "unknown channel\n");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
$raw = stream_get_contents(STDIN);
|
||||||
|
$msg = json_decode((string)$raw, true);
|
||||||
|
if (!is_array($msg) || empty($msg['id']) || !isset($msg['text'])) {
|
||||||
|
fwrite(STDERR, "malformed message\n");
|
||||||
|
exit(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rebuilt rather than stored as sent: a record is only ever the fields this store understands,
|
||||||
|
// so a sender running newer code cannot write keys this one will later hand to a page.
|
||||||
|
$clean = [
|
||||||
|
'id' => preg_replace('/[^a-f0-9]/i', '', (string)$msg['id']),
|
||||||
|
'ts' => (int)($msg['ts'] ?? time()),
|
||||||
|
'from' => preg_replace('/[^a-z0-9]/i', '', strtolower((string)($msg['from'] ?? ''))),
|
||||||
|
'kind' => in_array($msg['kind'] ?? 'msg', ['msg', 'question', 'notice'], true) ? $msg['kind'] : 'msg',
|
||||||
|
'text' => mb_substr((string)$msg['text'], 0, 4000),
|
||||||
|
'style' => vv_nc_clean_style(is_array($msg['style'] ?? null) ? $msg['style'] : []),
|
||||||
|
];
|
||||||
|
if ($clean['id'] === '' || $clean['from'] === '') { fwrite(STDERR, "malformed message\n"); exit(2); }
|
||||||
|
|
||||||
|
exit(vv_nc_append($channel, $clean) ? 0 : 1);
|
||||||
@@ -22,6 +22,42 @@
|
|||||||
# passes are not counted. Running daily keeps that to whatever NPM rotates in a day.
|
# passes are not counted. Running daily keeps that to whatever NPM rotates in a day.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Incremental by byte offset, never a re-read.
|
||||||
|
# 475 MB across 41 files cannot be parsed on a page load, and re-parsing what was already
|
||||||
|
# counted would make each pass slower than the last. Each run records where it stopped and
|
||||||
|
# resumes there, so cost tracks new traffic rather than total traffic.
|
||||||
|
#
|
||||||
|
# A rotated log is detected, not assumed.
|
||||||
|
# A file smaller than the offset already recorded for it means NPM rotated it beneath us, so
|
||||||
|
# that file's offset restarts at zero and a rotation is counted. The totals are cumulative and
|
||||||
|
# are never reset by it — what rotated out between two passes is simply not counted, which is
|
||||||
|
# why this runs often enough to keep that gap to whatever NPM rotates in a day.
|
||||||
|
#
|
||||||
|
# Varaverk's own probes are excluded.
|
||||||
|
# uptime_probe.sh requests every proxied host every minute under Varaverk-Uptime/1.0. Counting
|
||||||
|
# those would make the monitor the busiest client of everything it monitors.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Read-only against NPM. Access logs are read and never rotated, truncated or removed — that is
|
||||||
|
# NPM's business, and a stats collector that deleted its own inputs would be unrecoverable.
|
||||||
|
#
|
||||||
|
# --dry-run parses and reports without writing the store or advancing any offset, so a dry run
|
||||||
|
# leaves the next real pass with exactly the same work to do.
|
||||||
|
#
|
||||||
|
# --reset forgets offsets and totals deliberately. It starts from the current end of each log,
|
||||||
|
# so it discards history rather than double-counting it.
|
||||||
|
#
|
||||||
|
# The store is replaced atomically: written to a temp file, re-read and parsed to prove it is
|
||||||
|
# valid JSON, then renamed over the original. A pass that dies midway — or produces something
|
||||||
|
# unparseable — leaves the previous document untouched rather than a truncated one.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
Executable → Regular
+13
-13
@@ -15,19 +15,6 @@
|
|||||||
# Accepts --host=HOST2 to refresh a single host (used by the UI refresh button).
|
# Accepts --host=HOST2 to refresh a single host (used by the UI refresh button).
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# DESIGN PRINCIPLES
|
|
||||||
# ==============================================================================================
|
|
||||||
#
|
|
||||||
# Cache-First, Never Live on Page Load
|
|
||||||
# Remote arr APIs have non-trivial latency — calling them on every page view
|
|
||||||
# would make the arrs page slow and fragile. Writing to /tmp/vv_cache/ on a
|
|
||||||
# 2-hour schedule decouples page load time from network availability.
|
|
||||||
#
|
|
||||||
# Single-Host Refresh for UI
|
|
||||||
# The UI refresh button passes --host=HOSTN to update one host without waiting
|
|
||||||
# for the full 2-hour cycle. Keeps the cache fresh when a user requests it.
|
|
||||||
#
|
|
||||||
# ==============================================================================================
|
|
||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
@@ -44,6 +31,19 @@
|
|||||||
# back to live calls when a file is missing or stale.
|
# back to live calls when a file is missing or stale.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Cache-First, Never Live on Page Load
|
||||||
|
# Remote arr APIs have non-trivial latency — calling them on every page view
|
||||||
|
# would make the arrs page slow and fragile. Writing to /tmp/vv_cache/ on a
|
||||||
|
# 2-hour schedule decouples page load time from network availability.
|
||||||
|
#
|
||||||
|
# Single-Host Refresh for UI
|
||||||
|
# The UI refresh button passes --host=HOSTN to update one host without waiting
|
||||||
|
# for the full 2-hour cycle. Keeps the cache fresh when a user requests it.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
|||||||
@@ -281,12 +281,23 @@ echo "━━━ $ICON_GEAR Step 4: Update master.conf paths ━━━"
|
|||||||
NEW_MASTER="$DST/Configurations/master.conf"
|
NEW_MASTER="$DST/Configurations/master.conf"
|
||||||
if [[ "$DRY_RUN" == false ]]; then
|
if [[ "$DRY_RUN" == false ]]; then
|
||||||
if [[ -f "$NEW_MASTER" ]]; then
|
if [[ -f "$NEW_MASTER" ]]; then
|
||||||
sed -i "s|^\(\s*TARGET_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST\"|" "$NEW_MASTER"
|
# Only rewrite a path that is a literal. These three are derived from SCRIPTS_DIR in
|
||||||
sed -i "s|^\(\s*DATA_DIR\s*=\s*\)\"[^\"]*\"|\1\"$DST/data\"|" "$NEW_MASTER"
|
# current confs, which load_config.sh exports from its own location — so after the move
|
||||||
sed -i "s|^\(\s*STATE_DIR\s*=\s*\)\"[^\"]*\"|\1\"${DST}/data/state\"|" "$NEW_MASTER"
|
# they already point at the new install and hardcoding them here would turn a
|
||||||
echo " TARGET_DIR → $DST ✅"
|
# self-correcting value back into one that has to be maintained. Older confs still carry
|
||||||
echo " DATA_DIR → $DST/data ✅"
|
# literals, so the substitution stays for them.
|
||||||
echo " STATE_DIR → $DST/data/state ✅"
|
_mig_path() { # $1=key $2=new value
|
||||||
|
if grep -qE "^\s*$1\s*=\s*\"[^\"]*\\\$\{?(SCRIPTS_DIR|DATA_DIR)" "$NEW_MASTER"; then
|
||||||
|
echo " $1 — already derived, left alone ✅"
|
||||||
|
else
|
||||||
|
sed -i "s|^\(\s*$1\s*=\s*\)\"[^\"]*\"|\1\"$2\"|" "$NEW_MASTER"
|
||||||
|
echo " $1 → $2 ✅"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
_mig_path TARGET_DIR "$DST"
|
||||||
|
_mig_path DATA_DIR "$DST/data"
|
||||||
|
_mig_path STATE_DIR "$DST/data/state"
|
||||||
|
unset -f _mig_path
|
||||||
else
|
else
|
||||||
error "master.conf not found at $NEW_MASTER"
|
error "master.conf not found at $NEW_MASTER"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -384,6 +395,24 @@ else
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Repoint the WebGUI at the new location ────────────────────────────────────
|
||||||
|
# The .plg rebuilds this symlink on every boot and now follows SCRIPTS_DIR, but a migration is
|
||||||
|
# not a boot. Without this the WebGUI keeps serving the old copy until the next restart, so a
|
||||||
|
# migration appears to work while every page, endpoint and pull-target is still the old tree.
|
||||||
|
WEB_DIR="/usr/local/emhttp/plugins/varaverk"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would repoint $WEB_DIR → $DST/Plugin/unraid"
|
||||||
|
elif [[ -d "$DST/Plugin/unraid" ]]; then
|
||||||
|
ln -sfn "$DST/Plugin/unraid" "$WEB_DIR"
|
||||||
|
if [[ "$(readlink -f "$WEB_DIR")" == "$(readlink -f "$DST/Plugin/unraid")" ]]; then
|
||||||
|
echo "WebGUI repointed → $DST/Plugin/unraid ✅"
|
||||||
|
else
|
||||||
|
warn "Could not repoint $WEB_DIR — the WebGUI will keep serving the old tree until reboot"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
warn "$DST/Plugin/unraid missing — WebGUI left pointing at the old tree"
|
||||||
|
fi
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────────────────────
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_DONE Migration complete ━━━━━"
|
echo "━━━━━ $ICON_DONE Migration complete ━━━━━"
|
||||||
|
|||||||
@@ -5,31 +5,49 @@
|
|||||||
// to reach it. Generated so the assistant can answer "how do I change X" with a path through
|
// to reach it. Generated so the assistant can answer "how do I change X" with a path through
|
||||||
// the pages instead of an instruction to open master.conf.
|
// the pages instead of an instruction to open master.conf.
|
||||||
//
|
//
|
||||||
// WHY THE ASSISTANT NEEDS THIS AT ALL
|
|
||||||
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
|
|
||||||
// useless if it were — a page is a pile of divs, not a description of itself — so the assistant
|
|
||||||
// has never had any way to know the UI exists. It could name a conf key and nothing more.
|
|
||||||
// pages/readme/*.md is the one directory the chunker classifies as kind='ui', which is why the
|
|
||||||
// output lands there and not in docs/.
|
|
||||||
//
|
|
||||||
// WHY IT IS GENERATED
|
|
||||||
// A hand-written map is a second description of the pages, and the moment a card moves it
|
|
||||||
// starts lying with total confidence — which is worse than saying nothing, because the
|
|
||||||
// assistant will repeat it. Everything here is derived from the same registries the pages
|
|
||||||
// themselves are built from: VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script,
|
|
||||||
// VV_UI_SECTION_SURFACES for the pages that show sections by subject, and the conf files for
|
|
||||||
// the settings and their controls.
|
|
||||||
//
|
|
||||||
// OPERATIONAL MODEL
|
// OPERATIONAL MODEL
|
||||||
// Hand-run, and re-run after adding a conf section, a script mapping or a settings surface.
|
// Reads the section registries and the conf files, resolves each setting to the page and card
|
||||||
// Writes exactly one file and nothing else.
|
// that renders it, and writes the whole map in one pass. Nothing is merged with what is
|
||||||
|
// already there — the output is derived entirely from the registries, so a stale entry cannot
|
||||||
|
// survive a rebuild.
|
||||||
//
|
//
|
||||||
// php Tools/ui_map_build.php write the map
|
// DESIGN PRINCIPLES
|
||||||
// php Tools/ui_map_build.php --check report what it would change, write nothing
|
// The assistant cannot see the UI any other way.
|
||||||
|
// The retrieval index reads git-tracked files. PHP body markup is not indexed and would be
|
||||||
|
// useless if it were — a page is a pile of divs, not a description of itself — so the
|
||||||
|
// assistant has never had any way to know the UI exists. It could name a conf key and
|
||||||
|
// nothing more. pages/readme/*.md is the one directory the chunker classifies as
|
||||||
|
// kind='ui', which is why the output lands there and not in docs/.
|
||||||
//
|
//
|
||||||
// Only sections that are genuinely reachable are listed. A section no page renders is reported
|
// Generated, because a hand-written map lies with confidence.
|
||||||
// at the end as unreachable rather than silently omitted — a setting with no route is a real
|
// A second description of the pages starts being wrong the moment a card moves, and that
|
||||||
// finding, and the map is the only place that would notice.
|
// is worse than saying nothing, because the assistant will repeat it. Everything here is
|
||||||
|
// derived from the same registries the pages themselves are built from:
|
||||||
|
// VV_SCRIPT_CONF_SECTIONS for what the Scheduler shows per script, VV_UI_SECTION_SURFACES
|
||||||
|
// for the pages that show sections by subject, and the conf files for the settings and
|
||||||
|
// their controls.
|
||||||
|
//
|
||||||
|
// An unreachable section is reported, never dropped.
|
||||||
|
// A section no page renders is listed at the end rather than silently omitted. A setting
|
||||||
|
// with no route through the UI is a real finding, and this map is the only thing that
|
||||||
|
// would ever notice.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Writes exactly one file, pages/readme/ui-map.md, and nothing else. No conf is modified, no
|
||||||
|
// page is touched, and the registries it reads are only read.
|
||||||
|
//
|
||||||
|
// --check reports what would change and writes nothing, so the map can be verified current in
|
||||||
|
// a commit without regenerating it.
|
||||||
|
//
|
||||||
|
// Generated output only. Nothing hand-edited belongs in ui-map.md — an edit there is lost on
|
||||||
|
// the next run, which is the correct behaviour for a derived file and the reason the header
|
||||||
|
// says so.
|
||||||
|
//
|
||||||
|
// RUNTIME MODES
|
||||||
|
// php Tools/ui_map_build.php write the map
|
||||||
|
// php Tools/ui_map_build.php --check report what it would change, write nothing
|
||||||
|
//
|
||||||
|
// Hand-run. Re-run after adding a conf section, a script mapping or a settings surface.
|
||||||
//
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/confform.php the section registries, the parser, and the inferred controls
|
// include/confform.php the section registries, the parser, and the inferred controls
|
||||||
|
|||||||
@@ -30,9 +30,13 @@
|
|||||||
// majority of the traffic it reports and bury whatever real use these hosts get.
|
// majority of the traffic it reports and bury whatever real use these hosts get.
|
||||||
//
|
//
|
||||||
// Bounded storage, decided up front.
|
// Bounded storage, decided up front.
|
||||||
// Per domain: lifetime counters, the last 60 samples for the strip, 48 hourly buckets and
|
// Per domain: lifetime counters, the last 60 samples for the strip, 48 hourly buckets,
|
||||||
// 30 daily ones. That is a fixed size — a per-minute probe kept as raw samples would be
|
// 30 daily ones and 12 monthly ones. That is a fixed size — a per-minute probe kept as raw
|
||||||
// fifty thousand rows a day and the file would be the problem instead of the answer.
|
// samples would be fifty thousand rows a day and the file would be the problem instead of
|
||||||
|
// the answer. Each tier is what one view on the Proxies tab draws: 24h, 7d, 30d, 12 months.
|
||||||
|
//
|
||||||
|
// Bounded in domains too: a record nothing has probed for VV_DOMAIN_DROP_DAYS is retired,
|
||||||
|
// so hosts removed or renamed in NPM do not accumulate for ever under the live ones.
|
||||||
//
|
//
|
||||||
// The domain list follows NPM, not a hand-kept list in conf.
|
// The domain list follows NPM, not a hand-kept list in conf.
|
||||||
// A host added in the Proxies tab starts being probed without anyone remembering to add it
|
// A host added in the Proxies tab starts being probed without anyone remembering to add it
|
||||||
@@ -53,7 +57,7 @@
|
|||||||
// CONFIGURATION
|
// CONFIGURATION
|
||||||
// UPTIME_PROBE_ENABLED master switch (default true)
|
// UPTIME_PROBE_ENABLED master switch (default true)
|
||||||
// UPTIME_PROBE_TIMEOUT seconds per domain (default 8)
|
// UPTIME_PROBE_TIMEOUT seconds per domain (default 8)
|
||||||
// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 900)
|
// UPTIME_PROBE_LIST_TTL seconds to reuse the cached domain list (default 300)
|
||||||
// VV_UPTIME_UA the User-Agent, matched by npm_access_stats.php
|
// VV_UPTIME_UA the User-Agent, matched by npm_access_stats.php
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
@@ -65,6 +69,15 @@ const VV_UPTIME_UA = 'Varaverk-Uptime/1.0';
|
|||||||
const VV_SAMPLES_KEEP = 60; // one hour at a one-minute cadence
|
const VV_SAMPLES_KEEP = 60; // one hour at a one-minute cadence
|
||||||
const VV_HOURS_KEEP = 48;
|
const VV_HOURS_KEEP = 48;
|
||||||
const VV_DAYS_KEEP = 30;
|
const VV_DAYS_KEEP = 30;
|
||||||
|
// A year as twelve monthly buckets rather than 365 daily ones. The yearly view is a shape — which
|
||||||
|
// months were bad — not a date lookup, and rolling the day buckets out to 365 would grow the file
|
||||||
|
// twelvefold to answer the same question at a resolution nothing displays.
|
||||||
|
const VV_MONTHS_KEEP = 12;
|
||||||
|
// A domain removed or renamed in NPM stops being probed, and its record would otherwise sit here
|
||||||
|
// for ever — the store only ever gained keys before this. 90 days rather than something tighter
|
||||||
|
// because a host switched off for a season is a normal thing here and its history should survive
|
||||||
|
// that; a domain nobody has served for a quarter is gone for good.
|
||||||
|
const VV_DOMAIN_DROP_DAYS = 90;
|
||||||
const VV_EVENTS_KEEP = 20;
|
const VV_EVENTS_KEEP = 20;
|
||||||
|
|
||||||
$dryRun = in_array('--dry-run', $argv, true);
|
$dryRun = in_array('--dry-run', $argv, true);
|
||||||
@@ -138,14 +151,15 @@ if ($status || $events) {
|
|||||||
strtoupper($e['to']), $e['detail'] ?? '');
|
strtoupper($e['to']), $e['detail'] ?? '');
|
||||||
exit(0);
|
exit(0);
|
||||||
}
|
}
|
||||||
printf("%-34s %-6s %8s %8s %8s %s\n", 'domain', 'state', '1h', '24h', '30d', 'since');
|
printf("%-34s %-6s %8s %8s %8s %8s %s\n", 'domain', 'state', '24h', '7d', '30d', '1y', 'since');
|
||||||
foreach ($doms as $d => $r) {
|
foreach ($doms as $d => $r) {
|
||||||
$h = vv_uptime_window($r['hours'] ?? [], 1);
|
$h24 = vv_uptime_window($r['hours'] ?? [], 24);
|
||||||
$h24 = vv_uptime_window($r['hours'] ?? [], 24);
|
$d7 = vv_uptime_window($r['days'] ?? [], 7);
|
||||||
$d30 = vv_uptime_window($r['days'] ?? [], 30);
|
$d30 = vv_uptime_window($r['days'] ?? [], 30);
|
||||||
printf("%-34s %-6s %8s %8s %8s %s\n", substr($d, 0, 34), $r['state'] ?? '-',
|
$y1 = vv_uptime_window($r['months'] ?? [], 12);
|
||||||
$h === null ? '-' : $h . '%', $h24 === null ? '-' : $h24 . '%',
|
printf("%-34s %-6s %8s %8s %8s %8s %s\n", substr($d, 0, 34), $r['state'] ?? '-',
|
||||||
$d30 === null ? '-' : $d30 . '%',
|
$h24 === null ? '-' : $h24 . '%', $d7 === null ? '-' : $d7 . '%',
|
||||||
|
$d30 === null ? '-' : $d30 . '%', $y1 === null ? '-' : $y1 . '%',
|
||||||
!empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-');
|
!empty($r['last_change']) ? date('m-d H:i', $r['last_change']) : '-');
|
||||||
}
|
}
|
||||||
printf("\n%d domains, last pass %s\n", count($doms),
|
printf("\n%d domains, last pass %s\n", count($doms),
|
||||||
@@ -171,7 +185,7 @@ try {
|
|||||||
echo "UPTIME_PROBE_ENABLED is false\n"; exit(0);
|
echo "UPTIME_PROBE_ENABLED is false\n"; exit(0);
|
||||||
}
|
}
|
||||||
$timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8));
|
$timeout = max(2, (int) ($v['UPTIME_PROBE_TIMEOUT'] ?? 8));
|
||||||
$listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 900));
|
$listTtl = max(60, (int) ($v['UPTIME_PROBE_LIST_TTL'] ?? 300));
|
||||||
|
|
||||||
$store = vv_uptime_read();
|
$store = vv_uptime_read();
|
||||||
if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); }
|
if (!$store) { echo "uptime.json is malformed — refusing to overwrite it\n"; exit(1); }
|
||||||
@@ -233,6 +247,7 @@ try {
|
|||||||
$doms = $store['domains'] ?? [];
|
$doms = $store['domains'] ?? [];
|
||||||
$hourKey = date('YmdH', $now);
|
$hourKey = date('YmdH', $now);
|
||||||
$dayKey = date('Ymd', $now);
|
$dayKey = date('Ymd', $now);
|
||||||
|
$monKey = date('Ym', $now);
|
||||||
|
|
||||||
foreach ($handles as $d => $ch) {
|
foreach ($handles as $d => $ch) {
|
||||||
$errno = curl_errno($ch);
|
$errno = curl_errno($ch);
|
||||||
@@ -249,7 +264,7 @@ try {
|
|||||||
|
|
||||||
$r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null,
|
$r = $doms[$d] ?? ['checks' => 0, 'up' => 0, 'down' => 0, 'state' => null,
|
||||||
'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [],
|
'last_change' => null, 'samples' => [], 'hours' => [], 'days' => [],
|
||||||
'events' => [], 'since' => $now];
|
'months' => [], 'events' => [], 'since' => $now];
|
||||||
$r['checks']++;
|
$r['checks']++;
|
||||||
$up ? $r['up']++ : $r['down']++;
|
$up ? $r['up']++ : $r['down']++;
|
||||||
$r['last_code'] = $code;
|
$r['last_code'] = $code;
|
||||||
@@ -274,7 +289,9 @@ try {
|
|||||||
if (count($r['samples']) > VV_SAMPLES_KEEP)
|
if (count($r['samples']) > VV_SAMPLES_KEEP)
|
||||||
$r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP);
|
$r['samples'] = array_slice($r['samples'], -VV_SAMPLES_KEEP);
|
||||||
|
|
||||||
foreach ([['hours', $hourKey, VV_HOURS_KEEP], ['days', $dayKey, VV_DAYS_KEEP]] as [$k, $key, $keep]) {
|
foreach ([['hours', $hourKey, VV_HOURS_KEEP],
|
||||||
|
['days', $dayKey, VV_DAYS_KEEP],
|
||||||
|
['months', $monKey, VV_MONTHS_KEEP]] as [$k, $key, $keep]) {
|
||||||
$b = $r[$k][$key] ?? ['u' => 0, 't' => 0];
|
$b = $r[$k][$key] ?? ['u' => 0, 't' => 0];
|
||||||
$b['t']++;
|
$b['t']++;
|
||||||
if ($up) $b['u']++;
|
if ($up) $b['u']++;
|
||||||
@@ -286,6 +303,18 @@ try {
|
|||||||
}
|
}
|
||||||
curl_multi_close($mh);
|
curl_multi_close($mh);
|
||||||
|
|
||||||
|
// Retire domains nothing has probed in a long time. Keyed on last_at rather than on absence
|
||||||
|
// from the current list, which matters when NPM is unreachable: the cached list keeps being
|
||||||
|
// probed, every domain keeps getting a last_at, and an NPM outage therefore cannot empty the
|
||||||
|
// store. Only a domain that genuinely left the list stops being stamped.
|
||||||
|
$cutoff = $now - (VV_DOMAIN_DROP_DAYS * 86400);
|
||||||
|
$dropped = [];
|
||||||
|
foreach ($doms as $d => $r) {
|
||||||
|
// A record with no last_at at all is kept. It should not be possible — every probe stamps
|
||||||
|
// it — and deleting on missing data is the wrong way round for something irreversible.
|
||||||
|
if (isset($r['last_at']) && $r['last_at'] < $cutoff) { unset($doms[$d]); $dropped[] = $d; }
|
||||||
|
}
|
||||||
|
|
||||||
$store['domains'] = $doms;
|
$store['domains'] = $doms;
|
||||||
$store['last_pass'] = $now;
|
$store['last_pass'] = $now;
|
||||||
|
|
||||||
@@ -293,6 +322,10 @@ try {
|
|||||||
microtime(true) - $t0, $changes ? '' : ' (no state changes)');
|
microtime(true) - $t0, $changes ? '' : ' (no state changes)');
|
||||||
foreach ($changes as $c) echo " $c\n";
|
foreach ($changes as $c) echo " $c\n";
|
||||||
foreach ($downList as $c) echo " DOWN $c\n";
|
foreach ($downList as $c) echo " DOWN $c\n";
|
||||||
|
// Named, not silent. Dropping a record throws away months of history, and a line in the log is
|
||||||
|
// the only trace that it was this and not the store being reset by something else.
|
||||||
|
foreach ($dropped as $d)
|
||||||
|
echo " RETIRED $d — not probed in " . VV_DOMAIN_DROP_DAYS . " days\n";
|
||||||
|
|
||||||
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
if ($dryRun) { echo "dry run — nothing written\n"; exit(0); }
|
||||||
if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); }
|
if (!vv_uptime_write($store)) { echo 'could not write ' . vv_uptime_path() . "\n"; exit(1); }
|
||||||
|
|||||||
@@ -25,12 +25,45 @@
|
|||||||
# that, this monitor becomes fifty thousand requests a day in the logs it reports on.
|
# that, this monitor becomes fifty thousand requests a day in the logs it reports on.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# Probe what is served, not what was visited.
|
||||||
|
# The domain list comes from NPM's own proxy hosts, so a host nobody has requested is measured
|
||||||
|
# exactly like a busy one. Deriving the list from traffic would leave the quietest hosts — the
|
||||||
|
# ones most likely to be broken without anyone noticing — permanently unmonitored.
|
||||||
|
#
|
||||||
|
# The probe is excluded from the statistics it feeds.
|
||||||
|
# Every request carries Varaverk-Uptime/1.0, which npm_access_stats.sh filters out. Without
|
||||||
|
# that the monitor would be the largest single source of traffic in the logs it reports on,
|
||||||
|
# and every access figure would be measuring this script.
|
||||||
|
#
|
||||||
|
# The wrapper holds no logic.
|
||||||
|
# Flags are forwarded verbatim and nothing is interpreted here. Two places that both know what
|
||||||
|
# --dry-run means is two places that can disagree about it.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
# OPERATIONAL SAFEGUARDS
|
||||||
|
# ==============================================================================================
|
||||||
|
#
|
||||||
|
# UPTIME_PROBE_ENABLED gates the whole run. Off means no probes and no writes.
|
||||||
|
#
|
||||||
|
# Bounded per domain by UPTIME_PROBE_TIMEOUT, so an unresponsive host costs one timeout rather
|
||||||
|
# than stalling the pass — the probes run in parallel, so one slow domain never delays the rest.
|
||||||
|
#
|
||||||
|
# --dry-run probes and reports without writing. --status and --events read stored history and
|
||||||
|
# probe nothing, so neither can alter what it is describing.
|
||||||
|
#
|
||||||
|
# Read-only against NPM. The domain list is read; no proxy host, certificate or setting is
|
||||||
|
# touched by anything in this path.
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
# RUNTIME MODES
|
# RUNTIME MODES
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
# uptime_probe.sh one pass
|
# uptime_probe.sh one pass
|
||||||
# uptime_probe.sh --dry-run probe and report, write nothing
|
# uptime_probe.sh --dry-run probe and report, write nothing
|
||||||
# uptime_probe.sh --status per-domain uptime table (1h / 24h / 30d)
|
# uptime_probe.sh --status per-domain uptime table (24h / 7d / 30d / 1y)
|
||||||
# uptime_probe.sh --events recent state changes, newest first
|
# uptime_probe.sh --events recent state changes, newest first
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
+302
-15
@@ -92,6 +92,37 @@ function vvEscAttr(s) {
|
|||||||
.replace(/</g,'<').replace(/>/g,'>');
|
.replace(/</g,'<').replace(/>/g,'>');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// What a failed fetch says.
|
||||||
|
//
|
||||||
|
// Every fetch in this plugin used to end in `.catch(() => {})` — 24 of them. That is not error
|
||||||
|
// handling, it is error deletion: the request fails, nothing renders, nothing is logged, and the
|
||||||
|
// surface either sits on "Loading…" forever or silently keeps showing stale numbers. The mesh chat
|
||||||
|
// spent an unknown amount of time "taking a minute to load" because a ReferenceError was thrown on
|
||||||
|
// every render and swallowed here; the fault named itself the moment a catch reported it.
|
||||||
|
//
|
||||||
|
// Console always, because a poller that drops one tick should not shout on screen. A target
|
||||||
|
// element when the caller has one, because a panel that will otherwise never fill has to say why.
|
||||||
|
//
|
||||||
|
// Global for the same reason vvEscHtml is: pages/*.php are included one at a time and each would
|
||||||
|
// otherwise carry its own copy, which is the arrangement that lets two of them drift.
|
||||||
|
function vvFetchErr(where, e, el) {
|
||||||
|
const msg = (e && e.message) ? e.message : String(e || 'request failed');
|
||||||
|
try { console.warn('[varaverk] ' + where + ' — ' + msg, e); } catch (_) {}
|
||||||
|
if (el) {
|
||||||
|
const n = (typeof el === 'string') ? document.getElementById(el) : el;
|
||||||
|
if (n) { n.textContent = where + ' failed: ' + msg; n.style.color = '#a05a2c'; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Throws on a non-2xx instead of handing HTML to JSON.parse. Unraid answers an expired session
|
||||||
|
// with a 302 to the login page, so without this the reported error is "Unexpected token '<'",
|
||||||
|
// which names the symptom and hides the cause.
|
||||||
|
function vvJson(r) {
|
||||||
|
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||||||
|
return r.json();
|
||||||
|
}
|
||||||
|
|
||||||
// A URL about to be put in href/src or handed to window.open. Anything that is not plainly http,
|
// A URL about to be put in href/src or handed to window.open. Anything that is not plainly http,
|
||||||
// https or a site-relative path becomes empty — javascript: is the one that matters, and an
|
// https or a site-relative path becomes empty — javascript: is the one that matters, and an
|
||||||
// allowlist is the only way to say that without chasing encodings. include/docs.php applies the
|
// allowlist is the only way to say that without chasing encodings. include/docs.php applies the
|
||||||
@@ -192,6 +223,224 @@ function vvPrompt(text, def, opts) {
|
|||||||
}, function (val) { resolve(val === false ? null : String(val)); });
|
}, function (val) { resolve(val === false ? null : String(val)); });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Mirror onboarding, shared by the Partnership tab and the first-run wizard ─────────────────
|
||||||
|
//
|
||||||
|
// Here for the reason stated above: only one pages/*.php is included per request, and both the
|
||||||
|
// wizard and the Partnership tab need to render the same panel and run the same job. A second
|
||||||
|
// copy in setup.php would drift from the one in partnership.php, and the panel encodes a detail
|
||||||
|
// that is easy to get wrong on a copy — the terminal command must carry the SERVING host's
|
||||||
|
// SCRIPTS_DIR, because it is pasted into a terminal on that machine.
|
||||||
|
//
|
||||||
|
// Not in js/varaverk.js: that loads below the tab include, and the wizard returns before it.
|
||||||
|
|
||||||
|
// Where Varaverk is installed on THIS host, read live rather than baked into the page.
|
||||||
|
//
|
||||||
|
// Two reasons it cannot be a render-time constant. The wizard can move it: choosing appdata in
|
||||||
|
// step 1 triggers a migration and step 2 renders in the same page load, so PHP's value names the
|
||||||
|
// pre-migration location. And each host chooses independently — the owner may be on flash while
|
||||||
|
// the mirror is on appdata — so a panel rendered on either side must ask, not assume.
|
||||||
|
//
|
||||||
|
// It matters because the value ends up in a command the operator pastes into a root terminal.
|
||||||
|
// api/setup.php?action=detect re-reads varaverk.cfg. Cached; the panels re-render often.
|
||||||
|
let _vvScriptsDir = null;
|
||||||
|
function vvScriptsDir(fallback) {
|
||||||
|
if (_vvScriptsDir) return Promise.resolve(_vvScriptsDir);
|
||||||
|
return fetch('/plugins/varaverk/api/setup.php?action=detect&_=' + Date.now())
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(d => (_vvScriptsDir = (d && d.scripts_dir) || fallback))
|
||||||
|
.catch(() => (_vvScriptsDir = fallback));
|
||||||
|
}
|
||||||
|
|
||||||
|
// api/run.php answers when a job is LAUNCHED, not finished. An empty body is never success —
|
||||||
|
// Unraid's CSRF guard exits with one, and so does a PHP fatal.
|
||||||
|
function _vvPtRun(id, extraArgs) {
|
||||||
|
const params = {id, manual: '1'};
|
||||||
|
if (extraArgs) params.extra_args = extraArgs;
|
||||||
|
return fetch('/plugins/varaverk/api/run.php', {method: 'POST', body: new URLSearchParams(params)})
|
||||||
|
.then(r => { if (!r.ok) throw new Error('HTTP ' + r.status); return r.text(); })
|
||||||
|
.then(text => {
|
||||||
|
if (!text.trim()) throw new Error('Empty response — request rejected before it reached run.php');
|
||||||
|
return JSON.parse(text);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function _vvJobProgressEl(btn) {
|
||||||
|
let el = btn.parentElement.querySelector('.vv-jobprog');
|
||||||
|
if (!el) {
|
||||||
|
el = document.createElement('div');
|
||||||
|
el.className = 'vv-jobprog';
|
||||||
|
el.style.cssText = 'font-size:10px;margin-top:5px;white-space:nowrap;';
|
||||||
|
btn.parentElement.appendChild(el);
|
||||||
|
}
|
||||||
|
el.innerHTML = '<span style="color:#4a9eff;">⟳ starting…</span>';
|
||||||
|
return el;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Polls the job's stat for liveness and its log for the step banner, so a launched job shows
|
||||||
|
// where it is instead of appearing to do nothing for minutes.
|
||||||
|
const _vvJobPoll = {};
|
||||||
|
function vvPtWatchJob(id, mountEl) {
|
||||||
|
if (_vvJobPoll[id]) clearInterval(_vvJobPoll[id]);
|
||||||
|
const enc = encodeURIComponent(id);
|
||||||
|
const tick = () => {
|
||||||
|
fetch(`/plugins/varaverk/api/status.php?id=${enc}&_=${Date.now()}`)
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(s => {
|
||||||
|
if (!s.ok) return;
|
||||||
|
if (s.status === 'running') {
|
||||||
|
return fetch(`/plugins/varaverk/api/log.php?id=${enc}&_=${Date.now()}`)
|
||||||
|
.then(r => r.json())
|
||||||
|
.then(l => {
|
||||||
|
const lines = (l.content || '').split('\n');
|
||||||
|
let step = '';
|
||||||
|
for (let i = lines.length - 1; i >= 0; i--) {
|
||||||
|
const m = lines[i].match(/━━━\s*(?:[^\s]+\s+)?(Step [^━]+?)\s*━━━/);
|
||||||
|
if (m) { step = m[1].trim(); break; }
|
||||||
|
}
|
||||||
|
mountEl.innerHTML = `<span style="color:#4a9eff;">⟳ running</span>`
|
||||||
|
+ (step ? ` <span style="color:#666;">· ${vvEscHtml(step)}</span>` : '');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
clearInterval(_vvJobPoll[id]); delete _vvJobPoll[id];
|
||||||
|
const col = s.status === 'ok' ? '#4caf50' : (s.status === 'warn' ? '#ff9800' : '#f44336');
|
||||||
|
const lbl = s.status === 'ok' ? 'complete ✅'
|
||||||
|
: (s.status === 'never_run' ? 'did not start ⚠' : `${s.status} (exit ${s.exit ?? '?'})`);
|
||||||
|
mountEl.innerHTML = `<span style="color:${col};">${lbl}</span>`
|
||||||
|
+ ` <a href="?tab=scheduler" class="localURL" style="color:#556;margin-left:6px;">log</a>`;
|
||||||
|
if (typeof _vvPtReload === 'function') _vvPtReload();
|
||||||
|
// The wizard does have something to refresh, and saying it did not is why the mirror's
|
||||||
|
// Join button stayed greyed at "Running" after the run finished. Worse, the mirror's own
|
||||||
|
// job takes about three seconds — it only notifies the owner — while the Phase 2 it
|
||||||
|
// triggers runs for minutes on the far side. So "this job is done" is not "the
|
||||||
|
// partnership is done", and a single reload here would still show an unfinished wizard.
|
||||||
|
// vvOnJobDone polls until the checklist actually turns.
|
||||||
|
if (typeof vvOnJobDone === 'function') vvOnJobDone(id, s);
|
||||||
|
})
|
||||||
|
.catch(() => {});
|
||||||
|
};
|
||||||
|
tick();
|
||||||
|
_vvJobPoll[id] = setInterval(tick, 4000);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function vvPtOnboard(btn) {
|
||||||
|
if (!await vvConfirm('Run full partnership_onboard.sh?\n\nRun on the MIRROR first, then on the OWNER.\n\nUse Phase 1 + Phase 2 buttons for step-by-step control.')) return;
|
||||||
|
btn.disabled = true;
|
||||||
|
btn.textContent = '⟳ Starting…';
|
||||||
|
// Stays disabled while it runs. A timed re-enable invited the second click whose lock refusal
|
||||||
|
// overwrote the live run's job record.
|
||||||
|
const prog = _vvJobProgressEl(btn);
|
||||||
|
_vvPtRun('Partnership/partnership_onboard.sh')
|
||||||
|
.then(d => {
|
||||||
|
if (!d.ok) throw new Error(d.error ?? 'Unknown error');
|
||||||
|
btn.textContent = '⟳ Running…';
|
||||||
|
vvPtWatchJob('Partnership/partnership_onboard.sh', prog);
|
||||||
|
})
|
||||||
|
.catch(e => {
|
||||||
|
prog.innerHTML = `<span style="color:#f44336;">failed to start — ${vvEscHtml(String(e.message || e))}</span>`;
|
||||||
|
btn.disabled = false;
|
||||||
|
btn.textContent = '▶ Onboard';
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mirror's two-step join. Step 1 is a terminal step on purpose: ssh_setup.sh runs ssh-copy-id,
|
||||||
|
// which prompts for the owner's root password on a first install, and a WebGUI button cannot
|
||||||
|
// answer a password prompt. Offering only a button here was offering the one route that cannot
|
||||||
|
// work — it failed on ssh-copy-id every time.
|
||||||
|
// opts: {ownerName, termBase, termCmd, phase, hasPartner}
|
||||||
|
function vvRenderMirrorOnboard(opts) {
|
||||||
|
const owner = vvEscHtml(opts.ownerName || 'the owner');
|
||||||
|
const dis = opts.hasPartner === false ? 'disabled style="opacity:.35;cursor:default;"' : '';
|
||||||
|
|
||||||
|
// Collapses at phase 2, not phase 1.
|
||||||
|
//
|
||||||
|
// HOST<n>_PHASE1_DONE means the OWNER finished its Phase 1 — conf pushed, network created,
|
||||||
|
// confs cached. It says nothing about whether THIS host's key was ever installed on the owner,
|
||||||
|
// which is the only thing Step 1 does. Gating on it hid the terminal command at precisely the
|
||||||
|
// moment the mirror still needed it: flag set by the owner's push, key not installed, panel
|
||||||
|
// showing a button that cannot work.
|
||||||
|
//
|
||||||
|
// Phase 2 means the partnership is actually established, so the instructions have genuinely
|
||||||
|
// stopped being needed. Until then showing them costs nothing — a step already done reads as a
|
||||||
|
// reminder, a step still needed and hidden is a dead end.
|
||||||
|
// Phase 2 runs on the OWNER and takes minutes — 4m05s on a measured run, longer as the arr
|
||||||
|
// library grows. The mirror's own job finishes in about three seconds, because all it does is
|
||||||
|
// send the notification, so for the rest of that window the screen showed the unchanged
|
||||||
|
// phase-1 panel with a greyed-out button and no statement that anything was happening
|
||||||
|
// elsewhere. That reads as a hang, and the reasonable response to a hang is to start clicking.
|
||||||
|
if ((opts.phase ?? 0) < 2 && opts.waiting) {
|
||||||
|
const mins = opts.waitingSince
|
||||||
|
? Math.floor((Date.now() - opts.waitingSince) / 60000) : 0;
|
||||||
|
const secs = opts.waitingSince
|
||||||
|
? Math.floor(((Date.now() - opts.waitingSince) % 60000) / 1000) : 0;
|
||||||
|
return `<div style="padding:12px 14px;background:#1a1200;border:1px solid #3a2800;border-radius:4px;">
|
||||||
|
<div style="display:flex;align-items:center;gap:10px;flex-wrap:wrap;">
|
||||||
|
<span style="font-size:18px;line-height:1;color:#ff9800;">⟳</span>
|
||||||
|
<div style="flex:1;min-width:220px;">
|
||||||
|
<div style="color:#ff9800;font-size:12px;font-weight:600;">
|
||||||
|
Onboard in progress — please wait</div>
|
||||||
|
<div style="color:#8a6a2a;font-size:10px;margin-top:3px;line-height:1.5;">
|
||||||
|
${owner} is running Phase 2: deploying containers, syncing auth data and
|
||||||
|
bootstrapping the arr libraries. This normally takes 4–6 minutes and nothing is
|
||||||
|
needed from you here. This page updates itself when it finishes.</div>
|
||||||
|
${opts.waitingSince ? `<div style="color:#6a5020;font-size:10px;margin-top:4px;
|
||||||
|
font-family:monospace;">elapsed ${mins}m ${String(secs).padStart(2,'0')}s</div>` : ''}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((opts.phase ?? 0) >= 2) {
|
||||||
|
// Done state. This used to return the same "Join partnership · ▶ Onboard" panel as the
|
||||||
|
// not-started state, under a comment saying it collapsed — so a fully onboarded mirror
|
||||||
|
// rendered as one that had never run, and the only honest reading of the screen was that
|
||||||
|
// nothing had happened. The button is gone rather than disabled: there is nothing left for
|
||||||
|
// the mirror to initiate, and Phase 2 is re-run from the owner.
|
||||||
|
return `<div style="padding:10px 12px;background:#0a0f0a;border:1px solid #1a3a1a;border-radius:4px;">
|
||||||
|
<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;">
|
||||||
|
<span style="font-size:10px;font-weight:600;color:#4caf50;background:#0a1a0a;
|
||||||
|
padding:2px 9px;border-radius:10px;border:1px solid #1a4a1a;">Partnership active</span>
|
||||||
|
<span style="font-size:9px;color:#444;">onboarded by ${owner} · nothing further to do here</span>
|
||||||
|
</div>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return `<div style="padding:10px 12px;background:#0d0d0d;border:1px solid #1e3a5a;border-radius:4px;">
|
||||||
|
<div style="display:flex;flex-direction:column;gap:10px;">
|
||||||
|
<div>
|
||||||
|
<div style="display:flex;align-items:center;gap:6px;margin-bottom:6px;">
|
||||||
|
<span style="font-size:9px;font-weight:700;color:#4a9eff;background:#0a1828;
|
||||||
|
padding:2px 8px;border-radius:10px;border:1px solid #1a3a5a;">Step 1</span>
|
||||||
|
<span style="font-size:11px;color:#888;">Install SSH key on ${owner}</span>
|
||||||
|
</div>
|
||||||
|
<div style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-bottom:4px;">
|
||||||
|
<a href="${opts.termBase}" target="_blank" class="localURL"
|
||||||
|
style="padding:3px 10px;background:#0e1a2a;color:#7ab;border:1px solid #1e3a5a;
|
||||||
|
border-radius:3px;text-decoration:none;font-size:10px;white-space:nowrap;">Open Terminal</a>
|
||||||
|
<code onclick="navigator.clipboard.writeText('${opts.termCmd}').then(()=>{this.style.color='#4caf50';setTimeout(()=>this.style.color='#444',1500)})"
|
||||||
|
style="font-size:9px;color:#444;background:#080808;padding:3px 8px;border-radius:3px;
|
||||||
|
border:1px solid #181818;flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis;
|
||||||
|
white-space:nowrap;cursor:pointer;" title="Click to copy">${opts.termCmd}</code>
|
||||||
|
</div>
|
||||||
|
<div style="font-size:9px;color:#2a2a2a;">Enter ${owner} root password when prompted · a button cannot answer that prompt</div>
|
||||||
|
</div>
|
||||||
|
<div style="border-top:1px solid #1a1a1a;padding-top:10px;">
|
||||||
|
<div style="display:flex;align-items:center;justify-content:space-between;gap:10px;">
|
||||||
|
<div>
|
||||||
|
<div style="display:flex;align-items:center;gap:6px;margin-bottom:2px;">
|
||||||
|
<span style="font-size:9px;font-weight:700;color:#4caf50;background:#0a1a0a;
|
||||||
|
padding:2px 8px;border-radius:10px;border:1px solid #1a3a1a;">Step 2</span>
|
||||||
|
<span style="font-size:11px;color:#888;">Join partnership</span>
|
||||||
|
</div>
|
||||||
|
<div style="font-size:9px;color:#333;">Notifies ${owner} to run Phase 2 · if key already installed</div>
|
||||||
|
</div>
|
||||||
|
<button class="vv-pt-action-btn run" onclick="vvPtOnboard(this)" ${dis}
|
||||||
|
style="font-size:11px;white-space:nowrap;">▶ Onboard</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<?php
|
<?php
|
||||||
@@ -210,25 +459,60 @@ unset($_master, $_h1m, $_host1_blank, $_my_hostid, $_conf_missing);
|
|||||||
|
|
||||||
// Determine active tab
|
// Determine active tab
|
||||||
$tab = $_GET['tab'] ?? 'monitor';
|
$tab = $_GET['tab'] ?? 'monitor';
|
||||||
$validTabs = ['monitor', 'scheduler', 'docker', 'watchdog', 'partnership', 'fallback', 'arrs', 'rsync', 'auth', 'settings'];
|
$validTabs = ['monitor', 'scheduler', 'watchdog', 'partnership', 'fallback', 'arrs', 'rsync', 'auth', 'settings'];
|
||||||
|
|
||||||
// The AI tab exists only on HOST1, and only while AI_ENABLED is true. Appended to $validTabs
|
// The AI tab exists only on the AI owner, and only while AI_ENABLED is true. Appended to
|
||||||
// rather than filtered out of it, so the check below rejects ?tab=ai server-side as well —
|
// $validTabs rather than filtered out of it, so the check below rejects ?tab=ai server-side as
|
||||||
// omitting the link is presentation, not access control, and api/ai.php refuses every action
|
// well — omitting the link is presentation, not access control, and api/ai.php refuses the
|
||||||
// on the same two conditions independently.
|
// owner-only actions independently.
|
||||||
//
|
//
|
||||||
// The host half is not a preference: HOST1 owns the GPU, the Ollama process and the index, and
|
// The host half is a deliberate split, not a technical limit. The mesh shares one AI: every node
|
||||||
// include/ai.php only ever reads the *local* {HOST}_OLLAMA_URL — there is no Tailscale resolver
|
// reaches the owner's model through include/ai_rpc.php, so an assistant works everywhere. What
|
||||||
// in the PHP layer the way there is in the shell. On any other host the tab could only render
|
// does not travel is this tab — it carries the bug reports, the index and the model configuration,
|
||||||
// and then fail its own health check.
|
// the surface where a wrong answer is expensive and the vocabulary assumes you built the thing.
|
||||||
// The tab is the owner-only surface — it carries the bug reports, the index and the model
|
// Someone running two containers on a node they were handed gets the assistant, not the machinery
|
||||||
// configuration. Assistant docks elsewhere use vv_ai_ui_on(), which every node with a reachable
|
// behind it.
|
||||||
// model passes.
|
//
|
||||||
|
// Assistant docks elsewhere use vv_ai_ui_on(), which every node in the mesh passes.
|
||||||
$_vv_ai = vv_ai_owner_ui_on();
|
$_vv_ai = vv_ai_owner_ui_on();
|
||||||
if ($_vv_ai) $validTabs[] = 'ai';
|
if ($_vv_ai) $validTabs[] = 'ai';
|
||||||
|
|
||||||
|
// ── Local pages ───────────────────────────────────────────────────────────────────────────────
|
||||||
|
// pages/local/ is gitignored, so whatever is in it belongs to this installation alone and never
|
||||||
|
// reaches the public mirror. This loader is the tracked half: a generic extension point that
|
||||||
|
// knows nothing about what it is loading.
|
||||||
|
//
|
||||||
|
// It exists because the alternative — a tracked `if (file_exists(pages/thing.php))` per private
|
||||||
|
// page — puts the name and purpose of every private page into the public repo, which defeats the
|
||||||
|
// point of keeping the page out of it.
|
||||||
|
//
|
||||||
|
// Discovered rather than configured: a conf key listing local pages would itself be a tracked
|
||||||
|
// file naming them, and an untracked one would be a second thing to keep in sync with the
|
||||||
|
// directory. The directory is the declaration.
|
||||||
|
//
|
||||||
|
// The label comes from a `// vv-local-page: Name` line in the first 2KB of the file, falling back
|
||||||
|
// to the capitalised id. Reading it out of the file keeps the page self-describing — nothing
|
||||||
|
// outside it has to be edited to add one.
|
||||||
|
$localPages = [];
|
||||||
|
foreach (glob("$pluginDir/pages/local/*.php") ?: [] as $_lp) {
|
||||||
|
$_id = basename($_lp, '.php');
|
||||||
|
// Ids are restricted and collisions rejected: $tab is user input that becomes an include
|
||||||
|
// path below, and a local page must never be able to shadow a real tab.
|
||||||
|
if (!preg_match('/^[a-z0-9][a-z0-9_-]{0,31}$/', $_id)) continue;
|
||||||
|
if (in_array($_id, $validTabs, true)) continue;
|
||||||
|
$_lbl = ucfirst($_id);
|
||||||
|
if (preg_match('/^\s*(?:\/\/|#)\s*vv-local-page:\s*(.+)$/m', (string)@file_get_contents($_lp, false, null, 0, 2048), $_m)) {
|
||||||
|
$_lbl = trim($_m[1]);
|
||||||
|
}
|
||||||
|
$localPages[$_id] = ['path' => $_lp, 'label' => $_lbl];
|
||||||
|
$validTabs[] = $_id;
|
||||||
|
}
|
||||||
|
unset($_lp, $_id, $_lbl, $_m);
|
||||||
|
|
||||||
if (!in_array($tab, $validTabs)) $tab = 'monitor';
|
if (!in_array($tab, $validTabs)) $tab = 'monitor';
|
||||||
$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'docker' => 'Docker', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Media Stack', 'rsync' => 'Rsync', 'auth' => 'Auth Stack', 'settings' => 'Settings', 'ai' => 'AI'];
|
$tabLabels = ['monitor' => 'Monitor', 'scheduler' => 'Scheduler', 'watchdog' => 'Watchdog', 'partnership' => 'Partnership', 'fallback' => 'FallBack', 'arrs' => 'Media Stack', 'rsync' => 'Rsync', 'auth' => 'Auth Stack', 'settings' => 'Settings', 'ai' => 'AI'];
|
||||||
|
foreach ($localPages as $_id => $_lp) $tabLabels[$_id] = $_lp['label'];
|
||||||
|
unset($_id, $_lp);
|
||||||
|
|
||||||
// Cache stamp for the stylesheet and script below. Both are served straight off the plugin
|
// Cache stamp for the stylesheet and script below. Both are served straight off the plugin
|
||||||
// directory at a path that never changes, so a browser holding an old copy keeps using it after
|
// directory at a path that never changes, so a browser holding an old copy keeps using it after
|
||||||
@@ -271,9 +555,12 @@ foreach (['css/varaverk.css', 'js/varaverk.js'] as $_vv_a) {
|
|||||||
<!-- Tab content -->
|
<!-- Tab content -->
|
||||||
<div id="vv-content">
|
<div id="vv-content">
|
||||||
<?php
|
<?php
|
||||||
$page = "$pluginDir/pages/$tab.php";
|
// Local pages resolve from the map built above, never by composing a path out of $tab —
|
||||||
|
// the map's keys were validated against a strict pattern, so nothing user-supplied
|
||||||
|
// reaches an include.
|
||||||
|
$page = isset($localPages[$tab]) ? $localPages[$tab]['path'] : "$pluginDir/pages/$tab.php";
|
||||||
if (file_exists($page)) include $page;
|
if (file_exists($page)) include $page;
|
||||||
else echo "<p>Page not found: $tab</p>";
|
else echo "<p>Page not found: " . htmlspecialchars($tab, ENT_QUOTES) . "</p>";
|
||||||
?>
|
?>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -203,6 +203,46 @@ platform_get_temp_thresholds() {
|
|||||||
echo "${hdd_hot:-45} ${hdd_max:-55} ${ssd_hot:-60} ${ssd_max:-70}"
|
echo "${hdd_hot:-45} ${hdd_max:-55} ${ssd_hot:-60} ${ssd_max:-70}"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# platform_get_cpu_temp
|
||||||
|
# Writes the CPU temperature in °C to stdout as a decimal, or nothing if no sensor answers.
|
||||||
|
# Returns 1 when nothing could be read, so a caller can tell "no sensor" from "cold".
|
||||||
|
#
|
||||||
|
# Two things make this harder than it looks, and both produced wrong numbers for months:
|
||||||
|
#
|
||||||
|
# 1. lm-sensors prints the alarm thresholds on the same line as the reading:
|
||||||
|
# Core 0: +63.0 C (high = +80.0 C, crit = +100.0 C)
|
||||||
|
# Anything that scrapes numbers off the whole line and takes the largest reports 100 —
|
||||||
|
# the critical threshold — as the current temperature, on every Intel box, forever. The
|
||||||
|
# parenthetical is stripped before a single digit is read.
|
||||||
|
#
|
||||||
|
# 2. On AMD, Tctl is not the die temperature. It is a control value carrying a fixed offset
|
||||||
|
# (+27°C on Threadripper), which is why HOST1 read 70 while the die was at 43. Tdie is the
|
||||||
|
# real measurement and is preferred wherever both are published.
|
||||||
|
#
|
||||||
|
# Preference order: Tdie, then Intel's package sensor, then the board's own CPU Temp, then the
|
||||||
|
# hottest individual core. The first label that answers wins; within a label the maximum is
|
||||||
|
# taken, because a multi-die part publishes one line per die and the hottest is the one that
|
||||||
|
# matters.
|
||||||
|
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
platform_get_cpu_temp() {
|
||||||
|
command -v sensors >/dev/null 2>&1 || return 1
|
||||||
|
|
||||||
|
local out label value
|
||||||
|
out=$(sensors 2>/dev/null | sed 's/(.*//') # drop "(high = ..., crit = ...)"
|
||||||
|
[[ -n "$out" ]] || return 1
|
||||||
|
|
||||||
|
for label in 'Tdie' 'Package id 0' 'CPU Temp' 'Core '; do
|
||||||
|
value=$(echo "$out" | grep -i "^[[:space:]]*${label}" \
|
||||||
|
| grep -oE '[+-]?[0-9]+\.[0-9]+' | sort -n | tail -1)
|
||||||
|
if [[ -n "$value" ]]; then
|
||||||
|
echo "${value#+}" # bash printf tolerates a leading +, PHP casts fine, awk does not
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||||
# platform_is_maintenance_running
|
# platform_is_maintenance_running
|
||||||
# Returns 0 if a parity check or sync is currently in progress.
|
# Returns 0 if a parity check or sync is currently in progress.
|
||||||
@@ -412,7 +452,12 @@ require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
|
|||||||
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
|
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
|
||||||
\$failed = 0;
|
\$failed = 0;
|
||||||
foreach (\$results as \$r) {
|
foreach (\$results as \$r) {
|
||||||
echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
|
// 'seed' means the partner had no master.conf anywhere — say so, because that is the case
|
||||||
|
// where the push carried the mesh's identity to a node that could not have known it.
|
||||||
|
\$what = (\$r['mode'] ?? '') === 'seed'
|
||||||
|
? 'seeded → ' . (\$r['path'] ?? '')
|
||||||
|
: 'pushed → ' . (\$r['path'] ?? '');
|
||||||
|
echo \$r['host'] . ': ' . (\$r['ok'] ? \$what : 'FAILED — ' . \$r['error']) . PHP_EOL;
|
||||||
if (!\$r['ok']) \$failed++;
|
if (!\$r['ok']) \$failed++;
|
||||||
}
|
}
|
||||||
exit(\$failed > 0 ? 1 : 0);
|
exit(\$failed > 0 ? 1 : 0);
|
||||||
@@ -424,10 +469,14 @@ exit(\$failed > 0 ? 1 : 0);
|
|||||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||||
# platform_push_setup_state
|
# platform_push_setup_state
|
||||||
# Pushes the Varaverk wizard setup state to all partners via the WebGUI PHP API.
|
# Pushes the Varaverk wizard setup state to all partners via the WebGUI PHP API.
|
||||||
# No-op if php is unavailable. Always returns 0.
|
# Returns 1 if any partner did not take it, 0 otherwise. No-op returning 0 if php is absent.
|
||||||
|
#
|
||||||
|
# This used to end with a bare `return 0` under the comment "Always returns 0", which meant
|
||||||
|
# every caller that wrote `platform_push_setup_state || X=false` was testing a constant.
|
||||||
|
# Wiring a failure branch onto a helper that cannot fail is worse than hardcoding the tick,
|
||||||
|
# because the code reads as though it checked.
|
||||||
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
# ──────────────────────────────────────────────────────────────────────────────────────────────
|
||||||
platform_push_setup_state() {
|
platform_push_setup_state() {
|
||||||
command -v php &>/dev/null || return 0
|
command -v php &>/dev/null || return 0
|
||||||
php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null
|
php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; exit(vv_push_setup_state() > 0 ? 1 : 0);" 2>/dev/null
|
||||||
return 0
|
|
||||||
}
|
}
|
||||||
|
|||||||
+57
-502
@@ -1,8 +1,14 @@
|
|||||||
<?php
|
<?php
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
// PURPOSE
|
// PURPOSE
|
||||||
// AI tab endpoint. Serves the status banner, starts a chat turn, and reports its progress —
|
// Browser entry point for the AI subsystem. Serves the status banner, starts a chat turn, and
|
||||||
// the token-and-poll contract behind the AI page.
|
// reports its progress — the token-and-poll contract behind the AI page and every assistant
|
||||||
|
// dock in the plugin.
|
||||||
|
//
|
||||||
|
// Transport only. The actions themselves live in include/ai_actions.php, which this shares with
|
||||||
|
// Tools/ai_rpc.php — the mesh entry point another node reaches over SSH. This file owns the
|
||||||
|
// things that are true of a browser request and nothing else: the CSRF-covered method split,
|
||||||
|
// the request trace, the master switch, and the routing decision.
|
||||||
//
|
//
|
||||||
// OPERATIONAL MODEL
|
// OPERATIONAL MODEL
|
||||||
// Generation takes 25-76 seconds on this hardware, so a turn is not answered in the request
|
// Generation takes 25-76 seconds on this hardware, so a turn is not answered in the request
|
||||||
@@ -30,34 +36,25 @@
|
|||||||
// its token, so the token is not guessable.
|
// its token, so the token is not guessable.
|
||||||
//
|
//
|
||||||
// OPERATIONAL SAFEGUARDS
|
// OPERATIONAL SAFEGUARDS
|
||||||
// Every action is refused off HOST1, and ask is refused when AI is disabled.
|
// The per-action safeguards live with the actions.
|
||||||
// AI_ENABLED gates the whole subsystem and vv_is_ai_host() gates the node; the tab is
|
// Token minting, hex path validation, per-message history validation, scope whitelisting
|
||||||
// hidden when either fails, but hiding a link is not access control and the endpoint is
|
// and the detached spawn are all in include/ai_actions.php, documented there, and apply
|
||||||
// reachable directly. The host check sits ahead of the dispatch and answers 404.
|
// identically to a browser request and a mesh request. Restating them here would be two
|
||||||
|
// copies to keep in step and one of them always losing.
|
||||||
//
|
//
|
||||||
// Every token is validated as hex before it composes a path.
|
// AI_ENABLED is checked before routing, not after.
|
||||||
// vv_ai_job_path() returns null for anything else, and each caller checks. That pattern
|
// It is this node's own switch. A mirror with AI off must not forward to the owner —
|
||||||
// is what confines reads and deletes to the job directory.
|
// honouring the toggle locally while quietly using someone else's model is not what the
|
||||||
|
// switch says it does.
|
||||||
//
|
//
|
||||||
// History is validated per message, not trusted as a blob.
|
// Refusal is a 404, not a redirect or an empty 200.
|
||||||
// Role must be user or assistant, content must be a non-empty string, and each is
|
// The owner-only actions answer 404 off the owner. Omitting a link is presentation; this
|
||||||
// truncated. A crafted history could otherwise inject a system role or push the context
|
// endpoint is reachable directly, so the gate is enforced server-side too.
|
||||||
// past the offload ceiling. chat_save applies the same validation, because a stored
|
|
||||||
// conversation is replayed into a later prompt when it is reopened — an unchecked role
|
|
||||||
// written there is an injection that survives a reload rather than one turn.
|
|
||||||
//
|
//
|
||||||
// The question is length-capped before it reaches a command line.
|
// A remote failure is reported as a remote failure.
|
||||||
// It is passed to the worker through escapeshellarg, but an unbounded string would still
|
// vv_ai_rpc() returns the owner's own JSON when it gets one and a named transport error
|
||||||
// consume the context budget the retrieved chunks need.
|
// when it does not. Neither is silently turned into an empty success — a mirror that
|
||||||
//
|
// cannot reach the owner must say so rather than render an empty banner.
|
||||||
// The worker is spawned detached with output discarded.
|
|
||||||
// nohup, stdin from /dev/null, stdout and stderr to /dev/null — the job file is the only
|
|
||||||
// channel. A worker holding the request's file descriptors would keep the connection
|
|
||||||
// open, defeating the point of returning a token.
|
|
||||||
//
|
|
||||||
// Stale job files are reaped on each ask.
|
|
||||||
// /tmp is tmpfs so they vanish on reboot, but a long-lived host would accumulate one
|
|
||||||
// file per question asked. Anything older than an hour is removed.
|
|
||||||
//
|
//
|
||||||
// REQUEST
|
// REQUEST
|
||||||
// GET ?action=stats banner payload
|
// GET ?action=stats banner payload
|
||||||
@@ -85,9 +82,9 @@
|
|||||||
// {"ok":false,"error":…}
|
// {"ok":false,"error":…}
|
||||||
//
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/ai.php vv_ai_stats(), vv_ai_config(), vv_ai_job_*()
|
// include/ai_actions.php vv_ai_dispatch() — every action, shared with the mesh entry point
|
||||||
// include/ai_repair.php the findings store — loaded only by the two actions that read it
|
// include/ai_rpc.php vv_ai_route(), vv_ai_rpc() — where an action runs, and the SSH hop
|
||||||
// Tools/ai_chat_worker.php the detached worker
|
// include/ai.php vv_ai_enabled(), reached through ai_actions.php
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
// First executable statement, deliberately dependency-free. A request that is rejected by the
|
// First executable statement, deliberately dependency-free. A request that is rejected by the
|
||||||
// CSRF prepend never reaches here and a request that dies inside the include never reaches the
|
// CSRF prepend never reaches here and a request that dies inside the include never reaches the
|
||||||
@@ -103,496 +100,54 @@ if (strpos($_SERVER['REQUEST_URI'] ?? '', 'action=poll') === false) {
|
|||||||
. ' len=' . ($_SERVER['CONTENT_LENGTH'] ?? '-') . "\n", FILE_APPEND | LOCK_EX);
|
. ' len=' . ($_SERVER['CONTENT_LENGTH'] ?? '-') . "\n", FILE_APPEND | LOCK_EX);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
header('Cache-Control: no-store, no-cache');
|
header('Cache-Control: no-store, no-cache');
|
||||||
require_once dirname(__DIR__) . '/include/ai.php';
|
require_once dirname(__DIR__) . '/include/ai_actions.php';
|
||||||
|
require_once dirname(__DIR__) . '/include/ai_rpc.php';
|
||||||
// History depth is per profile and still decided server-side rather than by the page — the page
|
|
||||||
// simply no longer carries a second copy of the numbers. include/ai_profiles.php holds them.
|
|
||||||
// Reasoning is not stored in history, so it does not compound.
|
|
||||||
const VV_AI_MAX_QUESTION = 4000; // characters
|
|
||||||
const VV_AI_MAX_HIST_MSG = 4000; // characters per retained message
|
|
||||||
const VV_AI_JOB_TTL = 3600; // seconds before a job file is reaped
|
|
||||||
|
|
||||||
$isPost = $_SERVER['REQUEST_METHOD'] === 'POST';
|
$isPost = $_SERVER['REQUEST_METHOD'] === 'POST';
|
||||||
$action = trim($isPost ? ($_POST['action'] ?? '') : ($_GET['action'] ?? 'stats'));
|
$action = trim($isPost ? ($_POST['action'] ?? '') : ($_GET['action'] ?? 'stats'));
|
||||||
|
|
||||||
// Request trace. There is no nginx access log on this host and the CSRF prepend exits with an
|
// Params merged rather than picked by method. The POST-only checks inside the dispatcher are what
|
||||||
// empty body, so without this there is no way to tell "the request never arrived" from "the
|
// enforce the CSRF contract; which superglobal a value arrived in is not a security property, and
|
||||||
// request arrived and failed" — which is exactly the ambiguity that made the first hang
|
// merging means a handler that reads one key does not care how the request was shaped.
|
||||||
// undiagnosable. Excludes poll, which would otherwise write a line per second per open tab.
|
$params = $_POST + $_GET;
|
||||||
function vv_ai_log(string $msg): void {
|
|
||||||
if (!is_dir('/var/log/varaverk')) return;
|
|
||||||
@file_put_contents('/var/log/varaverk/ai.log',
|
|
||||||
date('Y-m-d H:i:s') . ' ' . $msg . "\n", FILE_APPEND | LOCK_EX);
|
|
||||||
}
|
|
||||||
if ($action !== 'poll') {
|
if ($action !== 'poll') {
|
||||||
vv_ai_log(sprintf('%s action=%s from=%s',
|
vv_ai_log(sprintf('%s action=%s from=%s',
|
||||||
$_SERVER['REQUEST_METHOD'] ?? '?', $action ?: '(none)',
|
$_SERVER['REQUEST_METHOD'] ?? '?', $action ?: '(none)',
|
||||||
$_SERVER['REMOTE_ADDR'] ?? '?'));
|
$_SERVER['REMOTE_ADDR'] ?? '?'));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Host gate, ahead of the dispatch rather than inside each action. Varaverk.page omits the tab
|
// Master switch, ahead of everything. With AI_ENABLED false the tab is not in the tab list and no
|
||||||
// on any host but HOST1, but a hidden link is not access control and this endpoint is reachable
|
// dock is rendered, so nothing in the UI can legitimately reach any action here — including the
|
||||||
// directly. Every action is refused rather than just the expensive ones — there is no such thing
|
// cheap reads, which would otherwise still answer with index and token figures for a subsystem the
|
||||||
// as a read this host is entitled to, since the index and the model are not here.
|
// operator has turned off. Not a 404: the switch is a setting, and the message names the setting.
|
||||||
if (!vv_ai_is_owner()) {
|
//
|
||||||
http_response_code(404);
|
// Checked before the routing below because it is this node's own switch either way. A mirror with
|
||||||
echo json_encode(['ok' => false, 'error' => 'AI is not available on this host']);
|
// AI off must not forward to the owner: the operator turned AI off on this box, and honouring that
|
||||||
exit;
|
// locally while quietly using someone else's model is not what the switch says it does.
|
||||||
}
|
|
||||||
|
|
||||||
// Master switch, on the same footing as the host gate rather than only in front of ask. With
|
|
||||||
// AI_ENABLED false the tab is not in the tab list and the scheduler dock is not rendered, so
|
|
||||||
// nothing in the UI can legitimately reach any action here — including the cheap reads, which
|
|
||||||
// would otherwise still answer with index and token figures for a subsystem the operator has
|
|
||||||
// turned off. Not a 404: the switch is a setting, and the message names the setting.
|
|
||||||
if (!vv_ai_enabled()) {
|
if (!vv_ai_enabled()) {
|
||||||
echo json_encode(['ok' => false, 'error' => 'AI_ENABLED is false — AI features are off']);
|
echo json_encode(['ok' => false, 'error' => 'AI_ENABLED is false — AI features are off']);
|
||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── stats ─────────────────────────────────────────────────────────────────────
|
// Routing. vv_ai_route() decides local, remote or refused for this action on this node; the three
|
||||||
// Served from the shared 'ai' cache that Tools/api_cache_writer.sh refreshes every minute, on
|
// outcomes and the reasoning behind each live in include/ai_rpc.php, next to the transport that
|
||||||
// the same terms as the monitor and arrs payloads. This action is polled every 30 seconds by
|
// carries them, rather than being restated here.
|
||||||
// every open tab and used to pay a full collection each time — around a second, most of it spent
|
$httpStatus = 200;
|
||||||
// waiting on Ollama and nvidia-smi — for numbers that only change when the writer next runs.
|
$route = vv_ai_route($action);
|
||||||
//
|
|
||||||
// ?live=1 bypasses it, for the case where something was just changed and the point is to see the
|
if ($route === VV_AI_ROUTE_DENY) {
|
||||||
// result. A missing cache always falls back to collecting, so the cache can never be the reason
|
http_response_code(404);
|
||||||
// the banner fails to render.
|
echo json_encode(['ok' => false, 'error' => 'This AI surface lives on the owner node only']);
|
||||||
if ($action === 'stats') {
|
|
||||||
echo json_encode(['ok' => true, 'stats' => vv_ai_stats_cached(isset($_GET['live']))]);
|
|
||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── tokens ────────────────────────────────────────────────────────────────────
|
$body = $route === VV_AI_ROUTE_REMOTE
|
||||||
// Separate from stats rather than folded into it. stats is polled every 30 seconds by every
|
? vv_ai_rpc($action, $params, $isPost, $httpStatus)
|
||||||
// open tab; this reads a file that grows without bound between prunes. The totals only move
|
: vv_ai_dispatch($action, $params, $isPost, $httpStatus);
|
||||||
// when a turn completes, and the page knows exactly when that happened, so it asks then.
|
|
||||||
if ($action === 'tokens') {
|
|
||||||
echo json_encode(['ok' => true, 'tokens' => vv_ai_token_stats()]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── poll ──────────────────────────────────────────────────────────────────────
|
if ($httpStatus !== 200) http_response_code($httpStatus);
|
||||||
if ($action === 'poll') {
|
echo json_encode($body);
|
||||||
$token = trim($_GET['token'] ?? '');
|
|
||||||
if (vv_ai_job_path($token) === null) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit;
|
|
||||||
}
|
|
||||||
$job = vv_ai_job_read($token);
|
|
||||||
if ($job === null) {
|
|
||||||
// The worker writes its first state after this request may already have arrived.
|
|
||||||
echo json_encode(['ok' => true, 'job' => ['status' => 'pending']]); exit;
|
|
||||||
}
|
|
||||||
echo json_encode(['ok' => true, 'job' => $job]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── memory ────────────────────────────────────────────────────────────────────
|
|
||||||
if ($action === 'memory_get') {
|
|
||||||
$m = vv_ai_memory_read();
|
|
||||||
echo json_encode(['ok' => true, 'memory' => $m['text'], 'chars' => $m['chars'],
|
|
||||||
'max' => vv_ai_memory_max(), 'exists' => $m['exists'],
|
|
||||||
'path' => vv_ai_memory_path()]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($action === 'memory_set') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
$r = vv_ai_memory_write((string)($_POST['memory'] ?? ''));
|
|
||||||
vv_ai_log('memory_set ' . ($r['ok'] ? 'ok chars=' . $r['chars'] : 'FAILED: ' . $r['error']));
|
|
||||||
echo json_encode($r + ['max' => vv_ai_memory_max()]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── learned-memory proposals ──────────────────────────────────────────────────
|
|
||||||
// The store the assistant files candidates into. Accepting is the only path by which model-written
|
|
||||||
// text reaches a prompt, and it is a POST so the CSRF prepend covers it.
|
|
||||||
if ($action === 'mem_proposals') {
|
|
||||||
require_once __DIR__ . '/../include/ai_memory_learn.php';
|
|
||||||
$m = vv_ai_memory_read('learned');
|
|
||||||
echo json_encode([
|
|
||||||
'ok' => true,
|
|
||||||
'enabled' => vv_ai_mem_learn_enabled(),
|
|
||||||
'auto' => vv_ai_mem_learn_auto(),
|
|
||||||
// The list states the gate as well as the rows: an empty list means "nothing proposed"
|
|
||||||
// when learning is on and "nothing is looking" when it is off, and those are different.
|
|
||||||
'open' => vv_ai_mem_list('open'),
|
|
||||||
'recent' => array_slice(vv_ai_mem_list(), 0, 25),
|
|
||||||
'learned' => ['chars' => $m['chars'], 'max' => vv_ai_memory_learned_max()],
|
|
||||||
]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($action === 'mem_proposal_action') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
require_once __DIR__ . '/../include/ai_memory_learn.php';
|
|
||||||
$id = trim($_POST['id'] ?? '');
|
|
||||||
$act = trim($_POST['act'] ?? '');
|
|
||||||
$r = vv_ai_mem_action($id, $act);
|
|
||||||
vv_ai_log(sprintf('mem_proposal id=%s act=%s %s', $id, $act,
|
|
||||||
$r['ok'] ? 'ok' : ('FAILED: ' . ($r['error'] ?? '?'))));
|
|
||||||
echo json_encode($r);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── stop ──────────────────────────────────────────────────────────────────────
|
|
||||||
// Cancels a generation in flight. Only ever signals ONE pid, verified to be the worker for this
|
|
||||||
// exact job — never a process group. Signalling a group is what took the WebGUI down on
|
|
||||||
// 2026-08-07, and no group kill is needed here: the worker is a single php process whose only
|
|
||||||
// child-like thing is an HTTP connection to Ollama, which dies with it.
|
|
||||||
//
|
|
||||||
// Whatever was already generated is kept. A turn stopped at 80% is usually stopped because the
|
|
||||||
// operator has seen enough, not because they want it discarded.
|
|
||||||
if ($action === 'stop') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
|
|
||||||
$token = trim($_POST['token'] ?? '');
|
|
||||||
if (vv_ai_job_path($token) === null) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$job = vv_ai_job_read($token);
|
|
||||||
if ($job === null) { echo json_encode(['ok' => false, 'error' => 'No such job']); exit; }
|
|
||||||
|
|
||||||
$status = (string)($job['status'] ?? '');
|
|
||||||
if ($status === 'done' || $status === 'error' || $status === 'stopped') {
|
|
||||||
echo json_encode(['ok' => true, 'already' => true, 'status' => $status]); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$pid = (int)($job['pid'] ?? 0);
|
|
||||||
// Below 2 is init or nonsense. A pid we cannot verify is a pid we do not signal.
|
|
||||||
$killed = false;
|
|
||||||
if ($pid >= 2) {
|
|
||||||
// Pid reuse is the reason for this: the recorded worker may have exited seconds ago and
|
|
||||||
// the number been handed to something else entirely. The cmdline must name both this
|
|
||||||
// worker and this job's own file before anything is signalled.
|
|
||||||
$cmdline = @file_get_contents("/proc/$pid/cmdline");
|
|
||||||
$cmdline = $cmdline === false ? '' : str_replace("\0", ' ', $cmdline);
|
|
||||||
if (strpos($cmdline, 'ai_chat_worker.php') !== false && strpos($cmdline, $token) !== false) {
|
|
||||||
$killed = @posix_kill($pid, SIGTERM);
|
|
||||||
// No escalation ladder. The worker holds no lock and writes the job file atomically,
|
|
||||||
// so there is no cleanup that a delay would protect — and a SIGKILL race could land
|
|
||||||
// between the temp write and the rename.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The job file is rewritten either way. If the pid could not be verified the worker is
|
|
||||||
// already gone, and the page still needs a terminal state instead of polling to its ceiling.
|
|
||||||
$job['status'] = 'stopped';
|
|
||||||
$job['stopped'] = true;
|
|
||||||
$job['answer'] = trim((string)($job['partial'] ?? $job['answer'] ?? ''));
|
|
||||||
unset($job['partial']);
|
|
||||||
@file_put_contents(vv_ai_job_path($token), json_encode($job));
|
|
||||||
|
|
||||||
vv_ai_log(sprintf('stop token=%s pid=%d signalled=%s kept=%d chars',
|
|
||||||
substr($token, 0, 12), $pid, $killed ? 'yes' : 'no', strlen($job['answer'])));
|
|
||||||
|
|
||||||
echo json_encode(['ok' => true, 'signalled' => $killed, 'kept' => strlen($job['answer'])]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── clear ─────────────────────────────────────────────────────────────────────
|
|
||||||
if ($action === 'clear') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
$p = vv_ai_job_path(trim($_POST['token'] ?? ''));
|
|
||||||
if ($p === null) { echo json_encode(['ok' => false, 'error' => 'Invalid token']); exit; }
|
|
||||||
if (file_exists($p)) @unlink($p);
|
|
||||||
echo json_encode(['ok' => true]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── chats ─────────────────────────────────────────────────────────────────────
|
|
||||||
// Stored conversations. Listing and reading are GET because they change nothing; saving and
|
|
||||||
// deleting are POST, so they ride Unraid's CSRF prepend like every other mutation here.
|
|
||||||
//
|
|
||||||
// Messages are validated per message on the way in, exactly as ask validates history and for
|
|
||||||
// the same reason: a stored chat is replayed into a later prompt when the operator reopens it,
|
|
||||||
// so a crafted role in the store would be an injection that survives a reload.
|
|
||||||
if ($action === 'chats') {
|
|
||||||
echo json_encode(['ok' => true, 'chats' => vv_ai_chats_list(), 'max' => vv_ai_chats_max()]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($action === 'chat_get') {
|
|
||||||
$chat = vv_ai_chat_read(trim($_GET['id'] ?? ''));
|
|
||||||
if ($chat === null) { echo json_encode(['ok' => false, 'error' => 'No such chat']); exit; }
|
|
||||||
echo json_encode(['ok' => true, 'chat' => $chat]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($action === 'chat_save') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
|
|
||||||
$profile = trim($_POST['profile'] ?? 'chat');
|
|
||||||
if (!vv_ai_profile_ok($profile)) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Unknown profile: ' . $profile]); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$clean = [];
|
|
||||||
$msgs = json_decode($_POST['messages'] ?? '[]', true);
|
|
||||||
if (is_array($msgs)) {
|
|
||||||
foreach ($msgs as $m) {
|
|
||||||
$role = $m['role'] ?? '';
|
|
||||||
$text = trim((string)($m['content'] ?? ''));
|
|
||||||
if (!in_array($role, ['user', 'assistant'], true) || $text === '') continue;
|
|
||||||
$clean[] = ['role' => $role, 'content' => mb_substr($text, 0, VV_AI_MAX_HIST_MSG)];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Capped at the deepest profile's window rather than that of the profile in hand. A chat
|
|
||||||
// saved under one profile can be reopened under another, and the reopened turn is trimmed
|
|
||||||
// again on the way back out by ask — so storing a little more than any single profile will
|
|
||||||
// send costs nothing and keeps the transcript readable.
|
|
||||||
$cap = vv_ai_profiles_max_turns() * 2;
|
|
||||||
if (count($clean) > $cap) $clean = array_slice($clean, -$cap);
|
|
||||||
|
|
||||||
// Whitelisted exactly as ask's is, and for the same reason: a scope is only ever a name from
|
|
||||||
// a page's own view state, it is stored and later replayed into a prompt, and anything
|
|
||||||
// richer than a file name is an instruction-injection surface for no benefit.
|
|
||||||
$scope = trim($_POST['scope'] ?? '');
|
|
||||||
if ($scope !== '' && !vv_ai_scope_ok($scope)) $scope = '';
|
|
||||||
|
|
||||||
$r = vv_ai_chat_save(trim($_POST['id'] ?? ''), $profile, $clean, $scope);
|
|
||||||
vv_ai_log('chat_save ' . ($r['ok'] ? 'ok id=' . substr($r['id'], 0, 12)
|
|
||||||
: 'FAILED: ' . $r['error']));
|
|
||||||
echo json_encode($r);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($action === 'chat_delete') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
$ok = vv_ai_chat_delete(trim($_POST['id'] ?? ''));
|
|
||||||
echo json_encode(['ok' => $ok, 'error' => $ok ? null : 'No such chat']);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── bugs / bug_close ──────────────────────────────────────────────────────────
|
|
||||||
// Reports the troubleshooter filed. Listing is a GET because it changes nothing; dismissing is
|
|
||||||
// a POST, like every other mutation in this plugin.
|
|
||||||
if ($action === 'bugs') {
|
|
||||||
echo json_encode(['ok' => true, 'bugs' => vv_ai_bugs_list(($_GET['all'] ?? '') !== '1')]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
if ($action === 'bug_close') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
$ok = vv_ai_bug_set_open(trim($_POST['id'] ?? ''), ($_POST['open'] ?? '0') === '1');
|
|
||||||
echo json_encode(['ok' => $ok]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// The report, rendered server-side. Read-only by design: what the operator reviews is byte for
|
|
||||||
// byte what gets sent, so approving one text and transmitting another is not possible. It is also
|
|
||||||
// the only renderer — the page used to build its own markdown, which is two formats to keep in
|
|
||||||
// step and one of them always losing.
|
|
||||||
if ($action === 'bug_report') {
|
|
||||||
$id = trim($_GET['id'] ?? '');
|
|
||||||
$bug = null;
|
|
||||||
foreach (vv_ai_bugs_list(false) as $b) if (($b['id'] ?? '') === $id) { $bug = $b; break; }
|
|
||||||
if (!$bug) { echo json_encode(['ok' => false, 'error' => 'no such report']); exit; }
|
|
||||||
|
|
||||||
$t = vv_ai_bug_targets();
|
|
||||||
$title = '[' . ($bug['component'] ?? '?') . '] ' . ($bug['summary'] ?? '');
|
|
||||||
echo json_encode([
|
|
||||||
'ok' => true,
|
|
||||||
'title' => $title,
|
|
||||||
'markdown' => vv_ai_bug_report($bug),
|
|
||||||
'targets' => $t,
|
|
||||||
// Built here because the repo name lives here. Length is the caller's problem to notice:
|
|
||||||
// GitHub truncates a very long query rather than refusing it, which would silently send a
|
|
||||||
// half report — so the page checks and falls back to the copy box.
|
|
||||||
'github' => 'https://github.com/' . $t['github_repo'] . '/issues/new?title='
|
|
||||||
. rawurlencode($title) . '&body=' . rawurlencode(vv_ai_bug_report($bug)),
|
|
||||||
]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sends to the operator's own Gitea, and only there. Never falls back to GitHub on failure: the
|
|
||||||
// two destinations are different people, and a silent substitution is how a report meant for a
|
|
||||||
// private backlog ends up public.
|
|
||||||
if ($action === 'bug_send_local') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
$id = trim($_POST['id'] ?? '');
|
|
||||||
$bug = null;
|
|
||||||
foreach (vv_ai_bugs_list(false) as $b) if (($b['id'] ?? '') === $id) { $bug = $b; break; }
|
|
||||||
if (!$bug) { echo json_encode(['ok' => false, 'error' => 'no such report']); exit; }
|
|
||||||
|
|
||||||
// Re-rendered from the store rather than taken from the request. The browser showed this text
|
|
||||||
// read-only; accepting a body from the page would make that guarantee decorative.
|
|
||||||
$r = vv_ai_bug_send_local('[' . ($bug['component'] ?? '?') . '] ' . ($bug['summary'] ?? ''),
|
|
||||||
vv_ai_bug_report($bug));
|
|
||||||
vv_ai_log(sprintf('bug_send_local id=%s %s', $id,
|
|
||||||
$r['ok'] ? 'ok ' . ($r['url'] ?? '') : 'failed: ' . ($r['error'] ?? '?')));
|
|
||||||
echo json_encode($r);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── findings / finding_action ─────────────────────────────────────────────────
|
|
||||||
// What the repair sweep found, and the operator's answer to it. include/ai_repair.php is pulled
|
|
||||||
// in here rather than at the top of the file: it is the largest include in the plugin and poll
|
|
||||||
// runs once a second per open tab, so it is loaded by the two actions that need it and by nothing
|
|
||||||
// else.
|
|
||||||
//
|
|
||||||
// Neither action is gated on AI_REPAIR_ENABLED. Findings filed while it was on do not stop being
|
|
||||||
// true when it goes off, and answering them — including saying "this was never a problem" — is
|
|
||||||
// exactly what an operator turning the feature off is likely to want to do first. The gate states
|
|
||||||
// are reported instead, so the card can say what is running rather than the endpoint pretending
|
|
||||||
// the store is empty.
|
|
||||||
if ($action === 'findings' || $action === 'finding_action') {
|
|
||||||
require_once dirname(__DIR__) . '/include/ai_repair.php';
|
|
||||||
|
|
||||||
if ($action === 'findings') {
|
|
||||||
// Closed findings are the history — what was dismissed, what a fix actually fixed — and
|
|
||||||
// they are asked for explicitly rather than shipped with every poll of the open list.
|
|
||||||
$rows = [];
|
|
||||||
$open = 0; $needs = 0;
|
|
||||||
foreach (vv_ai_findings_list(($_GET['all'] ?? '') === '1' ? [] : ['open', 'needs_operator']) as $f) {
|
|
||||||
$state = (string)($f['state'] ?? 'open');
|
|
||||||
if ($state === 'open') $open++;
|
|
||||||
elseif ($state === 'needs_operator') $needs++;
|
|
||||||
// The three things the page must not decide for itself: which actions this row
|
|
||||||
// offers, and what its state and kind mean in words.
|
|
||||||
$f['actions'] = vv_ai_finding_actions($f);
|
|
||||||
$f['state_label'] = VV_AI_FINDING_STATES[$state] ?? '';
|
|
||||||
$f['kind_label'] = VV_AI_FINDING_KINDS[(string)($f['kind'] ?? '')] ?? '';
|
|
||||||
$rows[] = $f;
|
|
||||||
}
|
|
||||||
echo json_encode(['ok' => true,
|
|
||||||
'repair' => ['enabled' => vv_ai_repair_enabled(),
|
|
||||||
'autofix' => vv_ai_repair_autofix_enabled(),
|
|
||||||
'last' => vv_ai_sweep_last()],
|
|
||||||
'findings' => $rows,
|
|
||||||
'counts' => ['open' => $open, 'needs_operator' => $needs, 'shown' => count($rows)]]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// POST, because fix writes conf through the guarded path and every other answer writes state.
|
|
||||||
// Which actions are legal for a given row is vv_ai_finding_apply_action()'s call, not this
|
|
||||||
// endpoint's — a tab left open overnight is holding buttons the store has moved past.
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
|
|
||||||
$fid = trim($_POST['id'] ?? '');
|
|
||||||
$act = trim($_POST['act'] ?? '');
|
|
||||||
$r = vv_ai_finding_apply_action($fid, $act, trim($_POST['note'] ?? ''));
|
|
||||||
vv_ai_log(sprintf('finding_action id=%s act=%s %s', $fid, $act,
|
|
||||||
$r['ok'] ? 'ok' : 'FAILED: ' . ($r['error'] ?? '?')));
|
|
||||||
echo json_encode($r);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── incident_add ──────────────────────────────────────────────────────────────
|
|
||||||
// Appends one operator-written "this was the fix" note against a scope. POST only, and the
|
|
||||||
// scope is whitelisted the same way ask's is — it is written to a file that later rides in a
|
|
||||||
// prompt, so it gets the same treatment as anything else that reaches the model.
|
|
||||||
if ($action === 'incident_add') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
echo json_encode(vv_ai_incident_add(
|
|
||||||
trim($_POST['scope'] ?? ''), trim($_POST['symptom'] ?? ''), trim($_POST['fix'] ?? '')));
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── ask ───────────────────────────────────────────────────────────────────────
|
|
||||||
if ($action === 'ask') {
|
|
||||||
if (!$isPost) { http_response_code(405); echo json_encode(['ok' => false, 'error' => 'POST only']); exit; }
|
|
||||||
|
|
||||||
$cfg = vv_ai_config();
|
|
||||||
if ($cfg['model'] === '') {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'No generation model configured']); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$question = trim($_POST['question'] ?? '');
|
|
||||||
if ($question === '') { echo json_encode(['ok' => false, 'error' => 'question is required']); exit; }
|
|
||||||
if (mb_strlen($question) > VV_AI_MAX_QUESTION) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'question exceeds ' . VV_AI_MAX_QUESTION . ' characters']); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$profile = trim($_POST['profile'] ?? 'varaverk');
|
|
||||||
if (!vv_ai_profile_ok($profile)) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Unknown profile: ' . $profile]); exit;
|
|
||||||
}
|
|
||||||
$maxTurns = vv_ai_profile_turns($profile);
|
|
||||||
|
|
||||||
// Where the caller is standing — "master.conf", "daily_sync_maintenance.sh", a log name.
|
|
||||||
// The scheduler page sends it so a question can say "this setting" and mean something; the
|
|
||||||
// AI tab sends nothing and the worker simply omits the location line.
|
|
||||||
//
|
|
||||||
// Whitelisted hard, not escaped and hoped for. It reaches the model as text, so anything
|
|
||||||
// richer than a file name is an instruction-injection surface for no benefit — a scope is
|
|
||||||
// only ever a name from this page's own view state.
|
|
||||||
$scope = trim($_POST['scope'] ?? '');
|
|
||||||
if ($scope !== '' && !vv_ai_scope_ok($scope)) $scope = '';
|
|
||||||
|
|
||||||
// The retrieval filter only means anything to the profile that retrieves.
|
|
||||||
$kind = vv_ai_profile_can($profile, 'kind_filter') ? trim($_POST['kind'] ?? '') : '';
|
|
||||||
if ($kind !== '' && !in_array($kind, VV_AI_KINDS, true)) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Unknown kind: ' . $kind]); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate per message rather than trusting the blob: a crafted history could otherwise
|
|
||||||
// inject a system role, or push the context past the offload ceiling.
|
|
||||||
$clean = [];
|
|
||||||
$hist = json_decode($_POST['history'] ?? '[]', true);
|
|
||||||
if (is_array($hist)) {
|
|
||||||
foreach ($hist as $m) {
|
|
||||||
$role = $m['role'] ?? '';
|
|
||||||
$text = trim((string)($m['content'] ?? ''));
|
|
||||||
if (!in_array($role, ['user', 'assistant'], true) || $text === '') continue;
|
|
||||||
$clean[] = ['role' => $role, 'content' => mb_substr($text, 0, VV_AI_MAX_HIST_MSG)];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (count($clean) > $maxTurns * 2) {
|
|
||||||
$clean = array_slice($clean, -($maxTurns * 2));
|
|
||||||
}
|
|
||||||
|
|
||||||
$dir = vv_ai_job_dir();
|
|
||||||
foreach (glob($dir . '/*.json') ?: [] as $old) {
|
|
||||||
if (time() - (int)@filemtime($old) > VV_AI_JOB_TTL) @unlink($old);
|
|
||||||
}
|
|
||||||
|
|
||||||
$token = bin2hex(random_bytes(16));
|
|
||||||
$jobFile = vv_ai_job_path($token);
|
|
||||||
$worker = dirname(__DIR__) . '/Tools/ai_chat_worker.php';
|
|
||||||
|
|
||||||
if (!file_exists($worker)) {
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'ai_chat_worker.php not found']); exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Not suppressed: if the job file cannot be written the worker has nowhere to report and
|
|
||||||
// the page polls a token that will never resolve — which looks exactly like a hang.
|
|
||||||
if (file_put_contents($jobFile, json_encode(['status' => 'pending'])) === false) {
|
|
||||||
vv_ai_log('ask FAILED — cannot write ' . $jobFile);
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Cannot write job file to ' . VV_AI_JOB_DIR]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
// setsid, not just nohup. nohup detaches from the terminal but leaves the child in the
|
|
||||||
// caller's process group — php-fpm's. That is the arrangement that took the WebGUI down on
|
|
||||||
// 2026-08-07 when a Stop signalled a group it did not own. Stop below signals one verified
|
|
||||||
// pid and never a group, so this is belt and braces, but it also means a php-fpm restart no
|
|
||||||
// longer takes a running generation with it.
|
|
||||||
$cmd = 'setsid nohup php ' . escapeshellarg($worker) . ' '
|
|
||||||
. escapeshellarg($jobFile) . ' '
|
|
||||||
. escapeshellarg($question) . ' '
|
|
||||||
. escapeshellarg(json_encode($clean)) . ' '
|
|
||||||
. escapeshellarg($kind) . ' '
|
|
||||||
. escapeshellarg(($_POST['think'] ?? '1') === '1' ? '1' : '0') . ' '
|
|
||||||
. escapeshellarg($profile) . ' '
|
|
||||||
. escapeshellarg($scope) . ' '
|
|
||||||
// Asked for per turn. Only meaningful on a profile holding web_search — the worker
|
|
||||||
// checks that, so a crafted web=1 against any other profile changes nothing.
|
|
||||||
. escapeshellarg(($_POST['web'] ?? '') === '1' ? '1' : '0')
|
|
||||||
. ' >/dev/null 2>&1 </dev/null &';
|
|
||||||
$out = []; $rc = 0;
|
|
||||||
exec($cmd, $out, $rc);
|
|
||||||
// Redacted before it is logged, for the same reason the stored transcript is: asking the
|
|
||||||
// assistant to set a credential means typing one, and ai.log is neither 0600 nor pruned.
|
|
||||||
vv_ai_log(sprintf('ask token=%s rc=%d profile=%s kind=%s q=%s',
|
|
||||||
substr($token, 0, 12), $rc, $profile, $kind ?: '-',
|
|
||||||
mb_substr(vv_ai_redact($question), 0, 80)));
|
|
||||||
|
|
||||||
echo json_encode(['ok' => true, 'token' => $token]);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
|
||||||
|
|||||||
@@ -126,6 +126,19 @@ function vv_uptime_window_api(array $buckets, int $n): ?float {
|
|||||||
return vv_auth_uptime_window($buckets, $n);
|
return vv_auth_uptime_window($buckets, $n);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One period of the history card: the rolled-up percentage, the drawable series, and how much of
|
||||||
|
// the window has actually been observed. All three come off the same buckets, so the number and
|
||||||
|
// the graph beside it can never disagree.
|
||||||
|
function vv_uptime_period(array $buckets, int $n, string $unit): array {
|
||||||
|
$series = vv_auth_uptime_series($buckets, $n, $unit);
|
||||||
|
return [
|
||||||
|
'pct' => vv_auth_uptime_window($buckets, $n),
|
||||||
|
'series' => $series,
|
||||||
|
'have' => count(array_filter($series, fn($v) => $v !== null)),
|
||||||
|
'want' => $n,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
function vv_auth_action_allowed(string $action): bool {
|
function vv_auth_action_allowed(string $action): bool {
|
||||||
$panel = VV_AUTH_ACTION_PANEL[$action] ?? null;
|
$panel = VV_AUTH_ACTION_PANEL[$action] ?? null;
|
||||||
// Unmapped actions are left to the existing "Unknown action" answer rather than being refused
|
// Unmapped actions are left to the existing "Unknown action" answer rather than being refused
|
||||||
@@ -190,6 +203,27 @@ if ($_SERVER['REQUEST_METHOD'] === 'GET') {
|
|||||||
'last_change' => $r['last_change'] ?? null,
|
'last_change' => $r['last_change'] ?? null,
|
||||||
'last_detail' => $r['last_detail'] ?? null,
|
'last_detail' => $r['last_detail'] ?? null,
|
||||||
'last_ms' => $r['last_ms'] ?? null,
|
'last_ms' => $r['last_ms'] ?? null,
|
||||||
|
// When this domain was last actually probed. The probe follows NPM's host list, so
|
||||||
|
// a domain removed there simply stops being visited while its record stays here —
|
||||||
|
// and a stored state of "down" then describes the moment it left, not today. The
|
||||||
|
// card needs this to tell "broken" from "no longer served".
|
||||||
|
'last_at' => $r['last_at'] ?? null,
|
||||||
|
// The history card below the table. One aligned series per period rather than the
|
||||||
|
// raw buckets: the client would otherwise have to re-derive calendar keys to know
|
||||||
|
// which of thirty slots a given day belongs in, and there would then be two
|
||||||
|
// implementations of that rule in two languages.
|
||||||
|
//
|
||||||
|
// `have` is what actually exists, so the card can say "collecting — 2 of 30 days"
|
||||||
|
// instead of printing a percentage computed from two days as though it were a
|
||||||
|
// month. The store began 2026-08-15; every window longer than a day is partial
|
||||||
|
// for a while, and a confident figure over a short sample is the one thing this
|
||||||
|
// card must not do.
|
||||||
|
'hist' => [
|
||||||
|
'h24' => vv_uptime_period($r['hours'] ?? [], 24, 'hour'),
|
||||||
|
'd7' => vv_uptime_period($r['days'] ?? [], 7, 'day'),
|
||||||
|
'd30' => vv_uptime_period($r['days'] ?? [], 30, 'day'),
|
||||||
|
'm12' => vv_uptime_period($r['months'] ?? [], 12, 'month'),
|
||||||
|
],
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
echo json_encode(['ok' => true, 'domains' => $out, 'last_pass' => $u['last_pass'] ?? null]);
|
echo json_encode(['ok' => true, 'domains' => $out, 'last_pass' => $u['last_pass'] ?? null]);
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ if (is_dir(LOG_DIR)) {
|
|||||||
if ($lastErr !== null)
|
if ($lastErr !== null)
|
||||||
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()];
|
$errors[] = ['script' => $script, 'line' => $lastErr, 'ts' => (int)$lf->getMTime()];
|
||||||
}
|
}
|
||||||
} catch (Exception $e) {}
|
} catch (Exception $e) { vv_log_error('api/board.php', 'log walk failed: ' . $e->getMessage()); }
|
||||||
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
usort($errors, fn($a, $b) => $b['ts'] - $a['ts']);
|
||||||
}
|
}
|
||||||
$out['errors'] = array_slice($errors, 0, 20);
|
$out['errors'] = array_slice($errors, 0, 20);
|
||||||
|
|||||||
@@ -91,11 +91,22 @@ $items = [];
|
|||||||
// ── Identity ──────────────────────────────────────────────────────────────────
|
// ── Identity ──────────────────────────────────────────────────────────────────
|
||||||
preg_match('/^\s*HOST1\s*=\s*"([^"]*)"/m', $master, $m1);
|
preg_match('/^\s*HOST1\s*=\s*"([^"]*)"/m', $master, $m1);
|
||||||
$host1 = trim($m1[1] ?? '');
|
$host1 = trim($m1[1] ?? '');
|
||||||
|
// "Server identity" is this server's, not HOST1's. It was hardcoded to read master.conf's HOST1
|
||||||
|
// and print "HOST1: <name>", so on the mirror the row that answers "which host am I" confidently
|
||||||
|
// named the other machine — and reported ok purely because the owner's slot was filled in, which
|
||||||
|
// says nothing about whether this host resolved to a slot at all.
|
||||||
|
$myName = '';
|
||||||
|
if ($hostId !== 'unknown') {
|
||||||
|
preg_match('/^\s*' . preg_quote($hostIdUp, '/') . '\s*=\s*"([^"]*)"/m', $master, $mSelf);
|
||||||
|
$myName = trim($mSelf[1] ?? '');
|
||||||
|
}
|
||||||
$items[] = [
|
$items[] = [
|
||||||
'id' => 'identity',
|
'id' => 'identity',
|
||||||
'label' => 'Server identity',
|
'label' => 'Server identity',
|
||||||
'ok' => !empty($host1),
|
'ok' => $hostId !== 'unknown' && !empty($myName),
|
||||||
'detail' => $host1 ? "HOST1: $host1" : 'HOST1 blank in master.conf',
|
'detail' => ($hostId !== 'unknown' && $myName)
|
||||||
|
? "$hostIdUp: $myName"
|
||||||
|
: 'This host does not match any HOST* entry in master.conf',
|
||||||
];
|
];
|
||||||
|
|
||||||
// ── Host conf ─────────────────────────────────────────────────────────────────
|
// ── Host conf ─────────────────────────────────────────────────────────────────
|
||||||
@@ -176,18 +187,41 @@ if (!empty($jfContainer)) {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── master.conf pull (partner servers only) ───────────────────────────────────────────────────
|
// ── master.conf delivery (partner servers only) ───────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// The question is whether this host HAS the owner's master.conf, not whether it went and
|
||||||
|
// fetched one. There are two ways it arrives and only one of them used to count:
|
||||||
|
//
|
||||||
|
// pull — this host ran 'Pull from HOST1', which sets master_conf_pulled in the setup state.
|
||||||
|
// push — HOST1 sent it during Phase 1 of onboarding, before this host even had Varaverk
|
||||||
|
// installed. Nothing on this side runs, so no flag is written here.
|
||||||
|
//
|
||||||
|
// Keying solely off the pull flag meant a node whose conf had been seeded by Phase 1 —
|
||||||
|
// populated, correct, naming both hosts — was told "Not yet pulled from HOST1" and offered a
|
||||||
|
// button to fetch what it already had.
|
||||||
|
//
|
||||||
|
// The push leaves its own evidence: Phase 1 ends by writing <THIS_HOST>_PHASE1_DONE into the
|
||||||
|
// owner's setup state and pushing that file here, so the flag can only be present on this
|
||||||
|
// machine because the owner completed a push AT this machine. That is the discriminator.
|
||||||
|
//
|
||||||
|
// A populated HOST1 line is deliberately NOT the test. The wizard's manual partner path writes
|
||||||
|
// HOST1 and HOST2 into the LOCAL template conf from what the operator typed, which would make
|
||||||
|
// a template look delivered and hide the one button that fixes it.
|
||||||
if ($hostId !== 'host1' && $hostId !== 'unknown') {
|
if ($hostId !== 'host1' && $hostId !== 'unknown') {
|
||||||
$state = vv_setup_state_read();
|
$state = vv_setup_state_read();
|
||||||
$pulled = !empty($state['master_conf_pulled']);
|
$pulled = !empty($state['master_conf_pulled']);
|
||||||
|
// Both spellings, for the same reason the partnership item below accepts both.
|
||||||
|
$pushed = !empty($state[$hostIdUp . '_PHASE1_DONE']) || !empty($state[$hostId . '_phase1_done']);
|
||||||
$items[] = [
|
$items[] = [
|
||||||
'id' => 'master_conf',
|
'id' => 'master_conf',
|
||||||
'label' => 'master.conf',
|
'label' => 'master.conf',
|
||||||
'ok' => $pulled,
|
'ok' => $pulled || $pushed,
|
||||||
'detail' => $pulled
|
'detail' => $pulled
|
||||||
? 'Synced from HOST1'
|
? "Pulled from $host1"
|
||||||
: ($host1 ? "Not yet pulled from $host1" : 'HOST1 hostname not set in master.conf'),
|
: ($pushed
|
||||||
'action' => (!$pulled && $host1) ? 'pull_master' : null,
|
? "Seeded by $host1 — identity read from it"
|
||||||
|
: ($host1 ? "Not yet pulled from $host1" : 'HOST1 hostname not set in master.conf')),
|
||||||
|
'action' => (!$pulled && !$pushed && $host1) ? 'pull_master' : null,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,6 +243,42 @@ if (!empty($host2)) {
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
$allOk = !in_array(false, array_column($items, 'ok'), true);
|
// ── Blocking vs deferrable ────────────────────────────────────────────────────
|
||||||
|
// Setup used to be complete only when every item was green, which made a media-server API key a
|
||||||
|
// gate on finishing onboarding. A mirror that runs no Emby, or runs one the operator has not got
|
||||||
|
// round to configuring, could never reach "complete" — and the checklist is what tells them
|
||||||
|
// whether they are done.
|
||||||
|
//
|
||||||
|
// Blocking items are the ones the mesh genuinely cannot work without: who this host is, its conf,
|
||||||
|
// its SSH key, the owner's master.conf, and the partnership itself. Everything else improves the
|
||||||
|
// install without being load-bearing, and can be dismissed with a recorded decision.
|
||||||
|
//
|
||||||
|
// Deferring is per item and reversible, stored in the setup state so it survives a reload. The
|
||||||
|
// item still shows — amber, "deferred" — rather than disappearing, because a dismissed item is a
|
||||||
|
// decision to revisit, not a thing that stopped being true.
|
||||||
|
$deferrable = ['api_key' => true, 'populated' => true, 'emby_key' => true, 'jellyfin_key' => true];
|
||||||
|
$state = vv_setup_state_read();
|
||||||
|
|
||||||
|
foreach ($items as &$item) {
|
||||||
|
$canDefer = !empty($deferrable[$item['id']]);
|
||||||
|
$item['blocking'] = !$canDefer;
|
||||||
|
$item['deferred'] = $canDefer && !$item['ok']
|
||||||
|
&& !empty($state['DEFER_' . strtoupper($item['id'])]);
|
||||||
|
if ($item['deferred']) {
|
||||||
|
$item['detail'] = ($item['detail'] ?? '') . ' — deferred';
|
||||||
|
$item['action'] = 'undefer';
|
||||||
|
} elseif ($canDefer && !$item['ok']) {
|
||||||
|
// Keep the item's real action as the primary; the UI offers defer alongside it.
|
||||||
|
$item['can_defer'] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unset($item);
|
||||||
|
|
||||||
|
// Complete when every blocking item is green and every deferrable one is green or dismissed.
|
||||||
|
$allOk = true;
|
||||||
|
foreach ($items as $i) {
|
||||||
|
if ($i['ok']) continue;
|
||||||
|
if ($i['blocking'] || empty($i['deferred'])) { $allOk = false; break; }
|
||||||
|
}
|
||||||
|
|
||||||
echo json_encode(['ok' => true, 'complete' => $allOk, 'host_id' => $hostId, 'items' => $items]);
|
echo json_encode(['ok' => true, 'complete' => $allOk, 'host_id' => $hostId, 'items' => $items]);
|
||||||
|
|||||||
@@ -1,89 +0,0 @@
|
|||||||
<?php
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
||||||
// PURPOSE
|
|
||||||
// Docker folder endpoint. GET returns the full container inventory with folder assignments;
|
|
||||||
// POST performs one folder operation — create, rename, delete, move a container, or sync
|
|
||||||
// the folder store against master.conf in either direction.
|
|
||||||
//
|
|
||||||
// OPERATIONAL MODEL
|
|
||||||
// Read and write share one URL, split on HTTP method. GET is the docker tab's poll and is
|
|
||||||
// always safe. POST carries an `action` naming exactly one library call. Anything that is
|
|
||||||
// neither GET nor POST is refused with 405 before a parameter is read.
|
|
||||||
//
|
|
||||||
// DESIGN PRINCIPLES
|
|
||||||
// The action list is a closed match expression, not a dispatch table.
|
|
||||||
// Six named actions map to six library functions. An unrecognised action falls to the
|
|
||||||
// default arm and returns an error — it cannot resolve to a callable, because no part
|
|
||||||
// of the request is ever used to build a function name.
|
|
||||||
//
|
|
||||||
// Grouping is metadata, never container control.
|
|
||||||
// This endpoint moves containers between folders in a JSON store. It does not start,
|
|
||||||
// stop, or recreate anything — that is docker_action.php, deliberately a separate file
|
|
||||||
// with a separate confirmation path in the UI.
|
|
||||||
//
|
|
||||||
// Validation belongs to the library.
|
|
||||||
// Folder ids and names are trimmed here and checked in include/docker.php, so the same
|
|
||||||
// rules apply whether a call arrives from this endpoint or from the conf sync.
|
|
||||||
//
|
|
||||||
// OPERATIONAL SAFEGUARDS
|
|
||||||
// Wrong method is refused with a status code, not just a body.
|
|
||||||
// 405 is set alongside the JSON error so a mistaken caller fails visibly rather than
|
|
||||||
// parsing an error object as data.
|
|
||||||
//
|
|
||||||
// Every parameter is optional and defaults to empty.
|
|
||||||
// ?? '' on all four inputs means a malformed POST reaches the library as blank strings
|
|
||||||
// and is rejected there, rather than raising an undefined-index warning into the JSON
|
|
||||||
// body and corrupting the response.
|
|
||||||
//
|
|
||||||
// The folder store is written atomically by the library.
|
|
||||||
// vv_dk_write_json() writes to .vv.tmp and rename()s, so a delete or move interrupted
|
|
||||||
// mid-write cannot leave a truncated store — which would scatter every container back
|
|
||||||
// to ungrouped.
|
|
||||||
//
|
|
||||||
// REQUEST
|
|
||||||
// GET full inventory, no parameters
|
|
||||||
// POST action=move_container container, folder_id
|
|
||||||
// POST action=create_folder name
|
|
||||||
// POST action=rename_folder folder_id, name
|
|
||||||
// POST action=delete_folder folder_id
|
|
||||||
// POST action=sync_conf_to_json | sync_json_to_conf no further parameters
|
|
||||||
//
|
|
||||||
// RESPONSE
|
|
||||||
// GET vv_dk_all() verbatim — containers, folders, icons, WebUI links
|
|
||||||
// POST {"ok":bool,"error":string|null} as returned by the invoked library call
|
|
||||||
//
|
|
||||||
// DEPENDS ON
|
|
||||||
// include/docker.php vv_dk_all(), vv_dk_move_container(), vv_dk_create_folder(),
|
|
||||||
// vv_dk_rename_folder(), vv_dk_delete_folder(),
|
|
||||||
// vv_dk_sync_conf_to_json(), vv_dk_sync_json_to_conf()
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
||||||
header('Content-Type: application/json');
|
|
||||||
require_once dirname(__DIR__) . '/include/docker.php';
|
|
||||||
|
|
||||||
if ($_SERVER['REQUEST_METHOD'] === 'GET') {
|
|
||||||
echo json_encode(vv_dk_all());
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
|
||||||
http_response_code(405);
|
|
||||||
echo json_encode(['ok' => false, 'error' => 'GET or POST only']);
|
|
||||||
exit;
|
|
||||||
}
|
|
||||||
|
|
||||||
$action = trim($_POST['action'] ?? '');
|
|
||||||
$container = trim($_POST['container'] ?? '');
|
|
||||||
$folderId = trim($_POST['folder_id'] ?? '');
|
|
||||||
$name = trim($_POST['name'] ?? '');
|
|
||||||
|
|
||||||
$result = match ($action) {
|
|
||||||
'move_container' => vv_dk_move_container($container, $folderId),
|
|
||||||
'create_folder' => vv_dk_create_folder($name),
|
|
||||||
'rename_folder' => vv_dk_rename_folder($folderId, $name),
|
|
||||||
'delete_folder' => vv_dk_delete_folder($folderId),
|
|
||||||
'sync_conf_to_json'=> vv_dk_sync_conf_to_json(),
|
|
||||||
'sync_json_to_conf'=> vv_dk_sync_json_to_conf(),
|
|
||||||
default => ['ok' => false, 'error' => 'Unknown action: ' . $action],
|
|
||||||
};
|
|
||||||
|
|
||||||
echo json_encode($result);
|
|
||||||
@@ -5,6 +5,13 @@
|
|||||||
// state, tier activation, handback strikes, covered container status — for the fallback
|
// state, tier activation, handback strikes, covered container status — for the fallback
|
||||||
// tab's 30s poll.
|
// tab's 30s poll.
|
||||||
//
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Computed fresh on every request, deliberately uncached. The inputs are small local state
|
||||||
|
// files that fallback.sh rewrites as it moves between states, so assembling them costs about
|
||||||
|
// nothing — and a cached fallback picture is the one kind of stale this tab must never serve.
|
||||||
|
// A page showing NORMAL because the answer was cached before the switch is worse than a page
|
||||||
|
// that took an extra moment to load.
|
||||||
|
//
|
||||||
// DESIGN PRINCIPLES
|
// DESIGN PRINCIPLES
|
||||||
// Thin transport. Every judgement about what a state file means lives in
|
// Thin transport. Every judgement about what a state file means lives in
|
||||||
// include/fallback.php. This file exists to give the browser a URL.
|
// include/fallback.php. This file exists to give the browser a URL.
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Start and stop the fallback daemon, its dry-run preview, and the failover test harness, on
|
||||||
|
// this host or on a partner. The Fallback tab's only write surface.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Every action shells out to the script that already owns the operation — fallback.sh --stop
|
||||||
|
// and fallback_test.sh --stop — rather than signalling PIDs from PHP. Those two know things
|
||||||
|
// this layer must not have to: which lock holds the process, how long to wait, and, for the
|
||||||
|
// test, that SIGKILL must never be used because only its EXIT trap removes the iptables rule
|
||||||
|
// it installed.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// The scripts own stopping; this endpoint owns routing.
|
||||||
|
// Duplicating the escalation logic here would put a second, divergent implementation of
|
||||||
|
// "how to stop fallback safely" in a language that cannot run its traps.
|
||||||
|
//
|
||||||
|
// Start is dry-run only.
|
||||||
|
// The live daemon is started by array_started.sh at array start, which is the one context
|
||||||
|
// where the tier delays and the state file mean what they say. A button that launched a
|
||||||
|
// live failover monitor mid-session — against a partner mid-maintenance, from a page
|
||||||
|
// someone was reading — is not a button this page should have. Previewing is safe and is
|
||||||
|
// what the page is for.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// POST only. Unraid's CSRF token injector is jQuery-only, so a native fetch() GET would fail
|
||||||
|
// silently anyway; making these POST means a link or prefetch cannot stop a daemon.
|
||||||
|
//
|
||||||
|
// host is matched against the configured host list, never used as a path or a shell word.
|
||||||
|
// The slot resolves to a hostname from conf, then to a Tailscale IP through the same
|
||||||
|
// unambiguous-prefix resolver the rest of the mesh uses. A value that does not name a
|
||||||
|
// configured host is refused before anything runs.
|
||||||
|
//
|
||||||
|
// setsid, not nohup+&, for the dry run.
|
||||||
|
// A backgrounded child stays in php-fpm's process group, and the group kill that ends a
|
||||||
|
// request takes it with it. setsid detaches it into its own session so it survives the
|
||||||
|
// response — the same fix the Scheduler's Stop button needed.
|
||||||
|
//
|
||||||
|
// Output is captured and returned, not discarded.
|
||||||
|
// These scripts report refusals in words — "did not exit within 30s", "NOT force-killing"
|
||||||
|
// — and a boolean would throw away the only explanation the operator gets.
|
||||||
|
//
|
||||||
|
// REQUEST
|
||||||
|
// POST action=stop|stop_test|start_dry|clear_lock host=<slot>
|
||||||
|
//
|
||||||
|
// RESPONSE
|
||||||
|
// {"ok":true,"output":string} action ran; output is the script's own report
|
||||||
|
// {"ok":false,"error":string} bad method, unknown action, or unresolvable host
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/fallback.php vv_pt_peer_lookup(), vv_pt_ts_peers(), vv_pt_ssh()
|
||||||
|
// Fallback/fallback.sh --stop
|
||||||
|
// Fallback/fallback_test.sh --stop
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
require_once dirname(__DIR__) . '/include/fallback.php';
|
||||||
|
|
||||||
|
header('Content-Type: application/json');
|
||||||
|
|
||||||
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'POST only']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$action = (string)($_POST['action'] ?? '');
|
||||||
|
$slot = strtolower((string)($_POST['host'] ?? ''));
|
||||||
|
|
||||||
|
$allowed = ['stop', 'stop_test', 'start_dry', 'clear_lock'];
|
||||||
|
if (!in_array($action, $allowed, true)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$hosts = vv_fb_known_hosts();
|
||||||
|
if (!isset($hosts[$slot])) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Unknown host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$isMe = ($slot === vv_detect_host());
|
||||||
|
$scripts = rtrim(SCRIPTS_DIR, '/');
|
||||||
|
|
||||||
|
// The command, as the script that owns the operation would be invoked by hand.
|
||||||
|
$cmds = [
|
||||||
|
'stop' => 'bash ' . escapeshellarg("$scripts/Fallback/fallback.sh") . ' --stop 2>&1',
|
||||||
|
'stop_test' => 'bash ' . escapeshellarg("$scripts/Fallback/fallback_test.sh") . ' --stop 2>&1',
|
||||||
|
// setsid so it outlives this request. No redirect: fallback.sh writes its own persistent
|
||||||
|
// log when stdout is not a terminal, so an ad-hoc preview and the array-start daemon leave
|
||||||
|
// their record in the same file rather than one going to tmpfs and vanishing on reboot.
|
||||||
|
'start_dry' => 'setsid bash ' . escapeshellarg("$scripts/Fallback/fallback.sh")
|
||||||
|
. ' --dry-run --log > /dev/null 2>&1 < /dev/null & echo started',
|
||||||
|
'clear_lock' => 'rm -f /tmp/unraid_locks/fallback.lock /tmp/unraid_locks/fallback_test.lock && echo cleared',
|
||||||
|
];
|
||||||
|
|
||||||
|
if ($isMe) {
|
||||||
|
@mkdir('/tmp/varaverk', 0755, true);
|
||||||
|
$out = (string)shell_exec($cmds[$action]);
|
||||||
|
echo json_encode(['ok' => true, 'output' => trim($out)]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remote: same command, same script, over the SSH this file's neighbours already use.
|
||||||
|
$tsPeers = vv_pt_ts_peers();
|
||||||
|
$ts = vv_pt_peer_lookup($tsPeers, $hosts[$slot]);
|
||||||
|
$ip = $ts['ip'] ?? null;
|
||||||
|
|
||||||
|
$myId = strtoupper(vv_detect_host());
|
||||||
|
$sshKey = vv_fb_scalar(vv_read_conf_raw(vv_detect_host() . '.conf'), $myId . '_SSH_KEY');
|
||||||
|
|
||||||
|
if (!$ip || !$sshKey) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Partner not resolvable — no Tailscale IP or no SSH key']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The remote's SCRIPTS_DIR is not this host's: appdata mode on one side and flash on the other
|
||||||
|
// is the normal case on this mesh, so ask the partner where it keeps them.
|
||||||
|
$remoteDir = trim((string)vv_pt_ssh($ip, $sshKey,
|
||||||
|
'sed -n \'s/^SCRIPTS_DIR="\(.*\)"$/\1/p\' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null'));
|
||||||
|
if ($remoteDir === '') $remoteDir = '/boot/config/plugins/varaverk';
|
||||||
|
|
||||||
|
$remoteCmds = [
|
||||||
|
'stop' => "bash '$remoteDir/Fallback/fallback.sh' --stop 2>&1",
|
||||||
|
'stop_test' => "bash '$remoteDir/Fallback/fallback_test.sh' --stop 2>&1",
|
||||||
|
'start_dry' => "setsid bash '$remoteDir/Fallback/fallback.sh'"
|
||||||
|
. " --dry-run --log > /dev/null 2>&1 < /dev/null & echo started",
|
||||||
|
'clear_lock' => 'rm -f /tmp/unraid_locks/fallback.lock /tmp/unraid_locks/fallback_test.lock && echo cleared',
|
||||||
|
];
|
||||||
|
|
||||||
|
$out = vv_pt_ssh($ip, $sshKey, $remoteCmds[$action]);
|
||||||
|
echo json_encode(['ok' => true, 'output' => trim((string)$out)]);
|
||||||
@@ -0,0 +1,418 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Failover coverage: FALLBACK_<me>_TIER1-4 in this host's own conf — which of THIS host's
|
||||||
|
// containers the partner starts when this host goes dark, and in which delay tier.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// GET this host's containers plus current tier membership.
|
||||||
|
// POST action=cover tiers=<json {container: tier}> rewrites all four tier arrays.
|
||||||
|
//
|
||||||
|
// Originally the Coverage picker on the Partnership page (e8ee5b0), removed the same day in
|
||||||
|
// 1a836da — "they describe what the partner runs during an outage, which is the Fallback tab's
|
||||||
|
// subject" — and never rehomed, because that tab had nothing to host it. This is that card,
|
||||||
|
// rebuilt where it belongs, with the services half left behind: pushing XML templates to a
|
||||||
|
// mirror is an onboard concern, not a failover one.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// Edit the array fallback.sh actually reads, not a parallel one.
|
||||||
|
// The coverage tiers already exist and already carry the timing. A second "what to fail
|
||||||
|
// over" list would be a second answer to the same question, and the two would drift.
|
||||||
|
//
|
||||||
|
// A host edits only its OWN tiers, and the page says so.
|
||||||
|
// FALLBACK_<host>_TIER* lives in that host's conf and describes what someone else runs for
|
||||||
|
// it. Sparse checkout means this host does not have the partner's host*.conf at all — only
|
||||||
|
// the read-only RAM cache conf_sync fills — so an editor for the partner's coverage would
|
||||||
|
// be writing to a cache that the next sync overwrites. Configure HOST2's coverage from
|
||||||
|
// HOST2. This is the same trap that left the Watchdog card reporting a partner's lists as
|
||||||
|
// empty when they were merely somewhere else.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// POST only for writes, so Unraid's CSRF guard applies.
|
||||||
|
//
|
||||||
|
// Names are validated against containers this host runs, PLUS whatever the tiers already name.
|
||||||
|
// The conf legitimately holds entries for containers not present right now — removed,
|
||||||
|
// stopped, or renamed. Validating only against the running set would refuse to save a list
|
||||||
|
// the operator never touched. New names still have to be real; the guard is against
|
||||||
|
// inventing containers, not against keeping ones already recorded.
|
||||||
|
//
|
||||||
|
// A tier outside 1-4 is rejected, never clamped. Silently moving a container from tier 9 to
|
||||||
|
// tier 4 would give it a 24-hour delay nobody asked for.
|
||||||
|
//
|
||||||
|
// An absent array is refused, not appended. Writing a new block into an unknown position in a
|
||||||
|
// conf is how a setting ends up in the wrong section and stops being read.
|
||||||
|
//
|
||||||
|
// Writing an empty list is allowed — "cover nothing" is a legitimate choice and the only way
|
||||||
|
// to express it.
|
||||||
|
//
|
||||||
|
// REQUEST
|
||||||
|
// GET → current lists
|
||||||
|
// POST action=cover tiers={"Emby":1,...} → rewrite tiers 1-4
|
||||||
|
//
|
||||||
|
// RESPONSE
|
||||||
|
// {"ok":true,...} read payload, or {"ok":true,"counts":{...}} after a write
|
||||||
|
// {"ok":false,"error":string} validation or write failure, stated
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/confform.php vv_conf_edit(), vv_conf_last_error(), vv_parse_conf_list()
|
||||||
|
// include/common.php vv_docker_containers()
|
||||||
|
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf()
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
header('Content-Type: application/json');
|
||||||
|
require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc()
|
||||||
|
require_once dirname(__DIR__) . '/include/confform.php';
|
||||||
|
require_once dirname(__DIR__) . '/include/common.php';
|
||||||
|
|
||||||
|
$hostId = vv_detect_host();
|
||||||
|
$hostUp = strtoupper($hostId);
|
||||||
|
$myConf = $hostId . '.conf';
|
||||||
|
|
||||||
|
$TIERS = [1, 2, 3, 4];
|
||||||
|
$tierVar = fn(int $t) => "FALLBACK_{$hostUp}_TIER{$t}";
|
||||||
|
|
||||||
|
// ── Read ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||||
|
$raw = vv_read_conf_raw($myConf);
|
||||||
|
$mRaw = vv_read_conf_raw('master.conf');
|
||||||
|
|
||||||
|
$cover = [];
|
||||||
|
foreach ($TIERS as $t) {
|
||||||
|
// host*.conf first, master.conf second — installs that kept the tiers there still read.
|
||||||
|
$vals = vv_parse_conf_list($raw, $tierVar($t)) ?: vv_parse_conf_list($mRaw, $tierVar($t));
|
||||||
|
foreach ($vals as $c) { $c = trim($c); if ($c !== '') $cover[$c] = $t; }
|
||||||
|
}
|
||||||
|
|
||||||
|
$containers = [];
|
||||||
|
foreach (vv_docker_containers() as $c) {
|
||||||
|
$n = is_array($c) ? ($c['name'] ?? '') : (string)$c;
|
||||||
|
if ($n !== '') $containers[] = $n;
|
||||||
|
}
|
||||||
|
sort($containers, SORT_NATURAL | SORT_FLAG_CASE);
|
||||||
|
|
||||||
|
// Stacks are declared by the PARTNERSHIP OWNER and deployed to everyone, so on a mirror they
|
||||||
|
// are not in this host's conf at all — HOST2_PARTNERSHIP_AUTH_STACK is the shipped template,
|
||||||
|
// still commented out, while the eight auth containers it describes run there permanently
|
||||||
|
// because the owner put them there. Reading "this host's" stacks left the mirror's card
|
||||||
|
// showing every one of them as ordinary, selectable, uncovered.
|
||||||
|
//
|
||||||
|
// Owner's conf first, then this host's, unioned: on the owner the two are the same file, and
|
||||||
|
// a host that declares extras of its own still has them honoured. The owner's copy reaches a
|
||||||
|
// mirror through the conf_sync RAM cache, which vv_read_host_conf_raw() knows how to find.
|
||||||
|
$ownerSlot = strtolower(vv_parse_conf_scalar(vv_read_conf_raw('master.conf'), 'PARTNERSHIP_OWNER_HOST'));
|
||||||
|
$stackSrc = [];
|
||||||
|
foreach (array_unique(array_filter([$ownerSlot, $hostId])) as $slot) {
|
||||||
|
$stackSrc[strtoupper($slot)] = vv_read_host_conf_raw($slot);
|
||||||
|
}
|
||||||
|
|
||||||
|
$stackOf = [];
|
||||||
|
$byLower = [];
|
||||||
|
foreach ($containers as $n) $byLower[strtolower($n)] = $n;
|
||||||
|
foreach ($stackSrc as $id => $srcRaw) {
|
||||||
|
foreach ([
|
||||||
|
'auth' => "{$id}_PARTNERSHIP_AUTH_STACK",
|
||||||
|
'arrs' => "{$id}_PARTNERSHIP_ARR_STACK",
|
||||||
|
'services' => "{$id}_PARTNERSHIP_SERVICES_STACK",
|
||||||
|
] as $label => $var) {
|
||||||
|
foreach (vv_parse_conf_list($srcRaw, $var) as $xml) {
|
||||||
|
$n = preg_replace('/^my-|\.xml$/', '', trim($xml));
|
||||||
|
if ($n === '') continue;
|
||||||
|
// Only what this host actually runs. The owner's stack lists everything it
|
||||||
|
// deploys mesh-wide; a name with no container here is not "always up" here.
|
||||||
|
if (!isset($byLower[strtolower($n)])) continue;
|
||||||
|
$stackOf[$byLower[strtolower($n)]] = $label;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Named in a tier but not installed here. Reported rather than filtered: a tier entry for a
|
||||||
|
// container that does not exist is a line fallback.sh fails on during an outage, which is
|
||||||
|
// the worst possible moment to find a typo.
|
||||||
|
$missing = [];
|
||||||
|
$have = array_map('strtolower', $containers);
|
||||||
|
foreach (array_keys($cover) as $n) if (!in_array(strtolower($n), $have, true)) $missing[] = $n;
|
||||||
|
|
||||||
|
echo json_encode([
|
||||||
|
'ok' => true,
|
||||||
|
'host' => $hostUp,
|
||||||
|
'containers' => $containers,
|
||||||
|
'cover' => (object)$cover,
|
||||||
|
'stacks' => (object)$stackOf,
|
||||||
|
'missing' => $missing,
|
||||||
|
'tier_vars' => array_map($tierVar, $TIERS),
|
||||||
|
]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Write ────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
// ── Readiness: would a failover actually work right now ─────────────────────────────────────
|
||||||
|
// Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS
|
||||||
|
// these rows; it never produces them. A model must not be the thing that says failover is ready —
|
||||||
|
// that is precisely the class of answer this codebase keeps finding to be confidently wrong, and
|
||||||
|
// on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed,
|
||||||
|
// none of them present on the partner.
|
||||||
|
//
|
||||||
|
// Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the
|
||||||
|
// order a real outage would hit them.
|
||||||
|
if (($_POST['action'] ?? '') === 'readiness') {
|
||||||
|
$rows = [];
|
||||||
|
$add = function (string $id, string $label, string $verdict, string $detail, string $ask = '')
|
||||||
|
use (&$rows) {
|
||||||
|
// verdict: ok | warn | fail | unknown — unknown is never dressed up as ok
|
||||||
|
$rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict,
|
||||||
|
'detail' => $detail, 'ask' => $ask];
|
||||||
|
};
|
||||||
|
|
||||||
|
$me = vv_detect_host();
|
||||||
|
$ME = strtoupper($me);
|
||||||
|
$conf = vv_read_conf_raw('master.conf');
|
||||||
|
$hc = vv_read_conf_raw($me . '.conf');
|
||||||
|
|
||||||
|
// 1. is fallback even armed
|
||||||
|
$fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||||
|
$add('enabled', 'Fallback armed',
|
||||||
|
$fbEnabled === 'true' ? 'ok' : 'fail',
|
||||||
|
'FALLBACK_ENABLED=' . $fbEnabled,
|
||||||
|
'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?');
|
||||||
|
|
||||||
|
// 2. current state — anything but NORMAL means it is already doing something
|
||||||
|
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||||
|
$state = 'unknown';
|
||||||
|
if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) {
|
||||||
|
$state = $m[1];
|
||||||
|
}
|
||||||
|
// Same rule the node cards use: no state file plus a live daemon means the node has simply
|
||||||
|
// never transitioned, which is health, not ignorance. Reading the file alone gives a healthy
|
||||||
|
// node the same verdict as one whose daemon is dead.
|
||||||
|
$daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||||
|
$inferred = false;
|
||||||
|
if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; }
|
||||||
|
|
||||||
|
$add('state', 'State is NORMAL',
|
||||||
|
$state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'),
|
||||||
|
'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''),
|
||||||
|
'Fallback state is ' . $state . '. What does that mean and what should I check?');
|
||||||
|
|
||||||
|
// 3. coverage configured at all
|
||||||
|
$covered = [];
|
||||||
|
for ($t = 1; $t <= 4; $t++) {
|
||||||
|
foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c;
|
||||||
|
}
|
||||||
|
$add('coverage', 'Containers are covered',
|
||||||
|
$covered ? 'ok' : 'fail',
|
||||||
|
$covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier',
|
||||||
|
$covered
|
||||||
|
? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.'
|
||||||
|
: 'Nothing is listed in my fallback tiers — what would happen if this host went dark?');
|
||||||
|
|
||||||
|
// 4. THE one that was silently false — does the partner actually hold them
|
||||||
|
$cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json';
|
||||||
|
if (!is_readable($cache)) {
|
||||||
|
$add('present', 'Partner has the containers', 'unknown',
|
||||||
|
'never checked — run the presence check',
|
||||||
|
'How do I find out whether the partner actually has my covered containers?');
|
||||||
|
} else {
|
||||||
|
$j = json_decode((string)@file_get_contents($cache), true);
|
||||||
|
$miss = (array)($j['missing'] ?? []);
|
||||||
|
$age = time() - (int)@filemtime($cache);
|
||||||
|
$when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago';
|
||||||
|
$add('present', 'Partner has the containers',
|
||||||
|
$miss ? 'fail' : 'ok',
|
||||||
|
$miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): '
|
||||||
|
. implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '')
|
||||||
|
: 'all ' . count($covered) . ' present (' . $when . ')',
|
||||||
|
$miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6))
|
||||||
|
. '. What happens during a failover, and how do I fix it?' : '');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. host-specific networks that cannot be recreated on the partner
|
||||||
|
$wg = [];
|
||||||
|
foreach ($covered as $c) {
|
||||||
|
foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) {
|
||||||
|
$t = @file_get_contents($x);
|
||||||
|
if ($t === false || strpos($t, "<Name>$c</Name>") === false) continue;
|
||||||
|
if (preg_match('~<Network>(wg\d+)</Network>~', $t, $m)) $wg[] = "$c ({$m[1]})";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($wg) {
|
||||||
|
$add('wgnet', 'No tunnel-bound networks', 'warn',
|
||||||
|
implode(', ', $wg),
|
||||||
|
'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. handback writeback — invisible until the day it matters
|
||||||
|
$wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||||
|
$add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn',
|
||||||
|
'FALLBACK_RSYNC_ENABLED=' . $wb,
|
||||||
|
'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?');
|
||||||
|
|
||||||
|
// Overall verdict is the worst row, never an average. One failed check is a failed failover.
|
||||||
|
$order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3];
|
||||||
|
$worst = 'ok';
|
||||||
|
foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict'];
|
||||||
|
|
||||||
|
echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows,
|
||||||
|
'summary' => $worst === 'ok'
|
||||||
|
? 'Every check passed'
|
||||||
|
: ($worst === 'fail' ? 'A failover would NOT work as configured'
|
||||||
|
: 'Failover is configured but something needs a look')]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
// ── Push / remove / status: what the PARTNER actually holds ──────────────────────────────────
|
||||||
|
// Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the
|
||||||
|
// partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from
|
||||||
|
// the partner, so every tier would have failed on the first real outage. These three actions are
|
||||||
|
// how the card closes and inspects that gap.
|
||||||
|
//
|
||||||
|
// Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write;
|
||||||
|
// deploying a dozen containers onto another machine is neither, and a stray click should not be
|
||||||
|
// able to do it.
|
||||||
|
$_covAction = $_POST['action'] ?? '';
|
||||||
|
if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) {
|
||||||
|
$dir = rtrim(SCRIPTS_DIR, '/');
|
||||||
|
$script = $dir . '/Fallback/coverage_deploy.sh';
|
||||||
|
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
||||||
|
|
||||||
|
if (!is_file($script)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is read-only and fast enough to answer inline; the two that change the partner are
|
||||||
|
// dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other
|
||||||
|
// long operation here.
|
||||||
|
if ($_covAction === 'deploy_status') {
|
||||||
|
$out = [];
|
||||||
|
exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc);
|
||||||
|
$present = []; $missing = [];
|
||||||
|
foreach ($out as $line) {
|
||||||
|
if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1];
|
||||||
|
elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2];
|
||||||
|
}
|
||||||
|
echo json_encode([
|
||||||
|
'ok' => true,
|
||||||
|
'present' => $present,
|
||||||
|
'missing' => $missing,
|
||||||
|
// rc 2 means "ran fine, some are missing" — not a failure of the check itself.
|
||||||
|
'checked' => ($rc === 0 || $rc === 2),
|
||||||
|
]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!is_file($runner)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$flag = $_covAction === 'push' ? '--push' : '--remove';
|
||||||
|
$stat = '/var/log/varaverk/Fallback/coverage_deploy.json';
|
||||||
|
|
||||||
|
shell_exec('setsid /bin/bash ' . escapeshellarg($runner)
|
||||||
|
. ' ' . escapeshellarg('Fallback/coverage_deploy.sh')
|
||||||
|
. ' ' . escapeshellarg($script)
|
||||||
|
. ' --manual ' . escapeshellarg($flag)
|
||||||
|
. ' >/dev/null 2>&1 </dev/null &');
|
||||||
|
|
||||||
|
// Report what the record says, not that the command was issued — run_job.sh writes its stat
|
||||||
|
// file before running, so a live record is the difference between a job that started and one
|
||||||
|
// refused for already running, or killed by the NORMAL-state gate.
|
||||||
|
for ($i = 0; $i < 12; $i++) {
|
||||||
|
if (is_file($stat)) {
|
||||||
|
$j = json_decode((string)@file_get_contents($stat), true);
|
||||||
|
if (is_array($j) && ($j['status'] ?? '') === 'running' && time() - filemtime($stat) < 60) {
|
||||||
|
echo json_encode(['ok' => true, 'status' => 'running', 'action' => $_covAction]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
usleep(250000);
|
||||||
|
}
|
||||||
|
echo json_encode(['ok' => false,
|
||||||
|
'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
if (($_POST['action'] ?? '') !== 'cover') {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$known = [];
|
||||||
|
foreach (vv_docker_containers() as $c) {
|
||||||
|
$n = is_array($c) ? ($c['name'] ?? '') : (string)$c;
|
||||||
|
if ($n !== '') $known[strtolower($n)] = $n;
|
||||||
|
}
|
||||||
|
$existingRaw = vv_read_conf_raw($myConf);
|
||||||
|
foreach ($TIERS as $t) {
|
||||||
|
foreach (vv_parse_conf_list($existingRaw, $tierVar($t)) as $n) {
|
||||||
|
$n = trim($n);
|
||||||
|
if ($n !== '' && !isset($known[strtolower($n)])) $known[strtolower($n)] = $n;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Stack containers refused here, not only greyed out in the picker. A disabled select is a
|
||||||
|
// courtesy to the operator, not a constraint on the endpoint.
|
||||||
|
//
|
||||||
|
// Sourced from the OWNER's conf as well as this host's, for the same reason the read path is: on
|
||||||
|
// a mirror the stack it runs is the owner's declaration, and checking only the local conf would
|
||||||
|
// have let a mirror assign a fallback tier to a container that never stops.
|
||||||
|
$ownerSlotW = strtolower(vv_parse_conf_scalar(vv_read_conf_raw('master.conf'), 'PARTNERSHIP_OWNER_HOST'));
|
||||||
|
$stackNames = [];
|
||||||
|
foreach (array_unique(array_filter([$ownerSlotW, $hostId])) as $slot) {
|
||||||
|
$srcRaw = vv_read_host_conf_raw($slot);
|
||||||
|
$id = strtoupper($slot);
|
||||||
|
foreach ([
|
||||||
|
"{$id}_PARTNERSHIP_AUTH_STACK",
|
||||||
|
"{$id}_PARTNERSHIP_ARR_STACK",
|
||||||
|
"{$id}_PARTNERSHIP_SERVICES_STACK",
|
||||||
|
] as $var) {
|
||||||
|
foreach (vv_parse_conf_list($srcRaw, $var) as $xml) {
|
||||||
|
$n = preg_replace('/^my-|\.xml$/', '', trim($xml));
|
||||||
|
if ($n !== '') $stackNames[strtolower($n)] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$map = json_decode((string)($_POST['tiers'] ?? ''), true);
|
||||||
|
if (!is_array($map)) { echo json_encode(['ok' => false, 'error' => 'tiers must be an object']); exit; }
|
||||||
|
|
||||||
|
$byTier = array_fill_keys($TIERS, []);
|
||||||
|
foreach ($map as $name => $tier) {
|
||||||
|
$t = (int)$tier;
|
||||||
|
if (!in_array($t, $TIERS, true)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => "Tier $tier is not 1-4 (for $name)"]); exit;
|
||||||
|
}
|
||||||
|
if (!isset($known[strtolower((string)$name)])) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => "No container named $name on this host"]); exit;
|
||||||
|
}
|
||||||
|
if (isset($stackNames[strtolower((string)$name)])) {
|
||||||
|
// Direction-neutral wording: on the owner this container is deployed TO the partner, on a
|
||||||
|
// mirror it was deployed HERE by the owner. Both mean the same thing for coverage — it
|
||||||
|
// runs on both nodes continuously, so there is nothing for a tier to start.
|
||||||
|
echo json_encode(['ok' => false,
|
||||||
|
'error' => "$name belongs to a partnership stack — it runs on both nodes continuously, so it cannot be given a fallback tier"]); exit;
|
||||||
|
}
|
||||||
|
$byTier[$t][] = $known[strtolower((string)$name)];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rewrites one `NAME=(` … `)` block in place, preserving the conf's leading indent.
|
||||||
|
$rewrite = function (string $cur, string $var, array $items): ?string {
|
||||||
|
$body = '';
|
||||||
|
foreach ($items as $i) $body .= " \"" . $i . "\"\n";
|
||||||
|
$pattern = '/^([ \t]*)' . preg_quote($var, '/') . '=\((?:[^)]*)\)/m';
|
||||||
|
if (!preg_match($pattern, $cur)) return null; // absent: refuse rather than append blind
|
||||||
|
return preg_replace_callback($pattern,
|
||||||
|
fn($m) => $m[1] . $var . "=(\n" . $body . $m[1] . ")", $cur, 1);
|
||||||
|
};
|
||||||
|
|
||||||
|
$ok = vv_conf_edit($myConf, function (string $cur) use ($byTier, $TIERS, $tierVar, $rewrite): ?string {
|
||||||
|
foreach ($TIERS as $t) {
|
||||||
|
$next = $rewrite($cur, $tierVar($t), $byTier[$t]);
|
||||||
|
if ($next === null) return null;
|
||||||
|
$cur = $next;
|
||||||
|
}
|
||||||
|
return $cur;
|
||||||
|
}, [], array_map($tierVar, $TIERS));
|
||||||
|
|
||||||
|
if (!$ok) { echo json_encode(['ok' => false, 'error' => vv_conf_last_error() ?: 'Write failed']); exit; }
|
||||||
|
|
||||||
|
// A partner holding the old list is a partner that will act on the old list.
|
||||||
|
vv_push_master_conf();
|
||||||
|
echo json_encode(['ok' => true, 'counts' => array_map('count', $byTier)]);
|
||||||
@@ -4,12 +4,19 @@
|
|||||||
// Active media sessions endpoint. Normalised now-playing across every Emby, Jellyfin and
|
// Active media sessions endpoint. Normalised now-playing across every Emby, Jellyfin and
|
||||||
// Plex instance configured for this host, for the monitor page's session panel.
|
// Plex instance configured for this host, for the monitor page's session panel.
|
||||||
//
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Local is the default and stays the cheap path: one call per media server configured on this
|
||||||
|
// host. Mesh adds one bounded SSH hop per partner and is only requested while the operator is
|
||||||
|
// looking at the mesh view, so a dashboard left open on the default costs exactly what it did
|
||||||
|
// before the scope existed.
|
||||||
|
//
|
||||||
// DESIGN PRINCIPLES
|
// DESIGN PRINCIPLES
|
||||||
// Thin transport. Discovery, per-server API dialects and normalisation all live in
|
// Thin transport. Discovery, per-server API dialects and normalisation all live in
|
||||||
// include/media.php; this file only sets the content type and encodes the result.
|
// include/media.php; this file only sets the content type and encodes the result.
|
||||||
//
|
//
|
||||||
// No parameters. Which servers to ask is derived from conf, not from the request, so the
|
// The request chooses a scope, never a target. scope= selects local or mesh; which servers
|
||||||
// browser cannot point this endpoint at an arbitrary URL.
|
// are asked, and which partners the mesh hop reaches, are both derived from conf. The browser
|
||||||
|
// can widen what it asks for, but it cannot point this endpoint at an arbitrary URL.
|
||||||
//
|
//
|
||||||
// OPERATIONAL SAFEGUARDS
|
// OPERATIONAL SAFEGUARDS
|
||||||
// Bounded by the library's 3s per-request timeout.
|
// Bounded by the library's 3s per-request timeout.
|
||||||
@@ -25,15 +32,25 @@
|
|||||||
// messages a client.
|
// messages a client.
|
||||||
//
|
//
|
||||||
// REQUEST
|
// REQUEST
|
||||||
// GET, no parameters
|
// GET this host's sessions
|
||||||
|
// GET ?scope=mesh every node's sessions, each row tagged with the host it is playing on
|
||||||
//
|
//
|
||||||
// RESPONSE
|
// RESPONSE
|
||||||
// vv_media_sessions() verbatim — a flat list of normalised sessions across all servers
|
// local {"scope":"local","sessions":[…],"server_names":[…],"server_count":N}
|
||||||
|
// mesh {"scope":"mesh","nodes":[…],"sessions":[…],"server_names":[…],"server_count":N}
|
||||||
|
//
|
||||||
|
// sessions is the normalised list; in the mesh scope each row also carries the host it is
|
||||||
|
// playing on. server_count counts media servers, not sessions. There is no error shape — an
|
||||||
|
// unreachable server contributes nothing; see OPERATIONAL SAFEGUARDS.
|
||||||
//
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/media.php vv_media_sessions()
|
// include/media.php vv_media_sessions(), vv_media_sessions_mesh()
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
require_once dirname(__DIR__) . '/include/media.php';
|
require_once dirname(__DIR__) . '/include/media.php';
|
||||||
|
|
||||||
echo json_encode(vv_media_sessions());
|
// Anything that is not the literal "mesh" is local. Fail-closed on the expensive path, matching
|
||||||
|
// how every other toggle in this plugin reads its value.
|
||||||
|
echo json_encode(($_GET['scope'] ?? '') === 'mesh'
|
||||||
|
? vv_media_sessions_mesh()
|
||||||
|
: vv_media_sessions() + ['scope' => 'local']);
|
||||||
|
|||||||
@@ -0,0 +1,132 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Start and stop the background media seed — Rsync/media_seed.sh, the multi-week first push
|
||||||
|
// of every DAILY_SYNC_SHARES entry to the partner. Phase 3 (partnership_onboard.sh
|
||||||
|
// --phase3-only) is how it is normally started; this endpoint is the direct control the
|
||||||
|
// Partnership tab uses to stop a running seed, and to start one without re-running the phase.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// The seed used to run inline inside partnership_onboard.sh, which meant the only control a
|
||||||
|
// person had over ~28 TB of transfer was killing the onboard. It is its own job now, with its
|
||||||
|
// own record at /var/log/varaverk/Rsync/media_seed.json, and these two actions are its whole
|
||||||
|
// control surface. Progress is read by include/partnership.php and rendered on the partner's
|
||||||
|
// card; nothing here reports progress.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// Stopping is safe by construction, and that is why a Stop button exists at all.
|
||||||
|
// rsync.sh runs --inplace --partial, so a stopped seed loses the file in flight, not the
|
||||||
|
// share, and a later start resumes rather than restarting. Anything else and the button
|
||||||
|
// would be a trap on a transfer measured in weeks.
|
||||||
|
//
|
||||||
|
// The job id is a literal.
|
||||||
|
// Nothing about which job runs comes from the request, so there is no path, no script
|
||||||
|
// name and no argument for a caller to influence. This endpoint can start exactly one
|
||||||
|
// script and signal exactly one recorded pid.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// POST only, for both actions. Unraid's CSRF guard is POST-only and jQuery-injected, so a
|
||||||
|
// GET here would be both unguarded and, from native fetch(), silently unauthenticated.
|
||||||
|
// See README-unraid.md.
|
||||||
|
//
|
||||||
|
// Stop signals the process group, not the pid.
|
||||||
|
// run_job.sh is dispatched under setsid, so its pid is its process group leader and
|
||||||
|
// kill -TERM -<pgid> reaches the rsync and the ssh beneath it. Signalling the pid alone
|
||||||
|
// would reap the wrapper and leave the transfer running with no record pointing at it.
|
||||||
|
//
|
||||||
|
// The recorded pid is verified to still be that job before it is signalled.
|
||||||
|
// A stale json from a run killed by a reboot can name a pid the kernel has since reused.
|
||||||
|
// /proc/<pid>/cmdline is checked for the seed script's own path first, so at worst this
|
||||||
|
// refuses to stop something; it cannot kill an unrelated process.
|
||||||
|
//
|
||||||
|
// Starting is delegated, not duplicated.
|
||||||
|
// run_job.sh already refuses to start a job that is running, and media_seed.sh takes its
|
||||||
|
// own lock. This endpoint does not re-implement either check — it dispatches and reports
|
||||||
|
// what the record then says.
|
||||||
|
//
|
||||||
|
// REQUEST
|
||||||
|
// POST action=start dispatch the seed detached
|
||||||
|
// POST action=stop terminate a running seed
|
||||||
|
//
|
||||||
|
// RESPONSE
|
||||||
|
// {"ok":true,"status":"running"} start: the job record went live within the wait window
|
||||||
|
// {"ok":true,"stopped":true} stop: the process group was signalled
|
||||||
|
// {"ok":false,"error":string} wrong method, unknown action, or nothing to act on
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// Rsync/media_seed.sh the job itself
|
||||||
|
// Plugin/unraid/run_job.sh the wrapper that writes the record
|
||||||
|
// include/config.php SCRIPTS_DIR
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
header('Content-Type: application/json');
|
||||||
|
require_once dirname(__DIR__) . '/include/config.php';
|
||||||
|
|
||||||
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'POST only']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$action = $_POST['action'] ?? '';
|
||||||
|
$dir = rtrim(SCRIPTS_DIR, '/');
|
||||||
|
$script = $dir . '/Rsync/media_seed.sh';
|
||||||
|
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
||||||
|
$stat = '/var/log/varaverk/Rsync/media_seed.json';
|
||||||
|
|
||||||
|
if ($action === 'start') {
|
||||||
|
if (!is_file($script) || !is_file($runner)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'media_seed.sh not found on this host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$cmd = 'setsid /bin/bash ' . escapeshellarg($runner)
|
||||||
|
. ' ' . escapeshellarg('Rsync/media_seed.sh')
|
||||||
|
. ' ' . escapeshellarg($script)
|
||||||
|
. ' --manual >/dev/null 2>&1 </dev/null &';
|
||||||
|
shell_exec($cmd);
|
||||||
|
|
||||||
|
// Report what the record says, not that the command was issued. run_job.sh writes its
|
||||||
|
// stat file before running the script, so a live record is the difference between a seed
|
||||||
|
// that started and one that was refused for already running or died on its gate check.
|
||||||
|
for ($i = 0; $i < 10; $i++) {
|
||||||
|
if (is_file($stat)) {
|
||||||
|
$j = json_decode((string)file_get_contents($stat), true);
|
||||||
|
if (is_array($j) && ($j['status'] ?? '') === 'running'
|
||||||
|
&& time() - filemtime($stat) < 60) {
|
||||||
|
echo json_encode(['ok' => true, 'status' => 'running']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
usleep(500000);
|
||||||
|
}
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Seed did not start — check Rsync/media_seed.log']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'stop') {
|
||||||
|
if (!is_file($stat)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'No seed has been run on this host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$j = json_decode((string)file_get_contents($stat), true);
|
||||||
|
$pid = (int)($j['pid'] ?? 0);
|
||||||
|
if (($j['status'] ?? '') !== 'running' || !$pid || !is_dir("/proc/$pid")) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Seed is not running']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// See "The recorded pid is verified" above — cmdline is NUL-separated, so the script path
|
||||||
|
// is matched against the raw bytes rather than a split.
|
||||||
|
$cmdline = @file_get_contents("/proc/$pid/cmdline") ?: '';
|
||||||
|
if (strpos($cmdline, 'media_seed.sh') === false) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Recorded PID is no longer the seed — record is stale']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
shell_exec('kill -TERM -' . $pid . ' 2>/dev/null');
|
||||||
|
usleep(400000);
|
||||||
|
if (is_dir("/proc/$pid")) shell_exec('kill -KILL -' . $pid . ' 2>/dev/null');
|
||||||
|
|
||||||
|
echo json_encode(['ok' => true, 'stopped' => true]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||||
@@ -112,6 +112,7 @@ if (vv_ai_ui_on()) {
|
|||||||
|
|
||||||
echo json_encode([
|
echo json_encode([
|
||||||
'system' => vv_system_info(),
|
'system' => vv_system_info(),
|
||||||
|
'varaverk' => vv_varaverk_state(),
|
||||||
'fallback' => vv_fallback_state(),
|
'fallback' => vv_fallback_state(),
|
||||||
'fallback_active' => vv_fallback_active(),
|
'fallback_active' => vv_fallback_active(),
|
||||||
'partner' => vv_partner_state(),
|
'partner' => vv_partner_state(),
|
||||||
|
|||||||
@@ -1,66 +0,0 @@
|
|||||||
<?php
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
||||||
// PURPOSE
|
|
||||||
// Remote node metrics endpoint. The monitor page's partner cards — CPU, memory, storage,
|
|
||||||
// uptime and container counts for every host other than this one.
|
|
||||||
//
|
|
||||||
// OPERATIONAL MODEL
|
|
||||||
// Split from monitor.php on cost, not on subject. Local metrics are cheap file reads;
|
|
||||||
// remote metrics are SSH round trips to every partner. Keeping them on separate URLs lets
|
|
||||||
// the page poll local stats often and remote stats rarely, and lets a dark partner slow
|
|
||||||
// only its own request. Served from a 1-hour cache by default; ?live bypasses it for the
|
|
||||||
// page's explicit refresh button.
|
|
||||||
//
|
|
||||||
// DESIGN PRINCIPLES
|
|
||||||
// The cache is the default and the live call is the exception.
|
|
||||||
// An hour is deliberately long. Partner hardware stats do not move fast enough to
|
|
||||||
// justify paying SSH latency on every page load, and the refresh button exists for the
|
|
||||||
// moment someone actually needs current numbers.
|
|
||||||
//
|
|
||||||
// Every payload carries its own timestamp.
|
|
||||||
// ts is written into the cached document, so the page can render the age rather than
|
|
||||||
// presenting hour-old numbers as current.
|
|
||||||
//
|
|
||||||
// The live path writes the cache too.
|
|
||||||
// A manual refresh benefits every subsequent visitor instead of being discarded.
|
|
||||||
//
|
|
||||||
// OPERATIONAL SAFEGUARDS
|
|
||||||
// Cache miss is distinguished from empty payload.
|
|
||||||
// vv_cache_read() returns null on a miss, expiry, or unparseable file, and the check is
|
|
||||||
// an explicit !== null. A legitimately empty result — the single-host case, where there
|
|
||||||
// are no remote hosts at all — is served from cache rather than being mistaken for a
|
|
||||||
// miss and forced onto the SSH path on every single poll.
|
|
||||||
//
|
|
||||||
// Read-only over SSH. The remote commands are stat collection only; nothing is started,
|
|
||||||
// stopped, or written on a partner.
|
|
||||||
//
|
|
||||||
// Unreachable partners degrade per node inside vv_remote_hosts_stats(), so one dark host
|
|
||||||
// cannot empty the other cards.
|
|
||||||
//
|
|
||||||
// HTTP caching is disabled even though the payload is cached server-side.
|
|
||||||
// The two are not the same lever. The server-side cache has an age the page can see and
|
|
||||||
// a bypass it can trigger; a browser or proxy cache has neither, and would defeat ?live
|
|
||||||
// entirely.
|
|
||||||
//
|
|
||||||
// REQUEST
|
|
||||||
// GET served from the 3600s cache when one is present
|
|
||||||
// GET ?live bypass the cache, collect fresh, and rewrite it
|
|
||||||
//
|
|
||||||
// RESPONSE
|
|
||||||
// {"remote_hosts":{…},"ts":epoch}
|
|
||||||
//
|
|
||||||
// DEPENDS ON
|
|
||||||
// include/monitor.php vv_remote_hosts_stats(), vv_cache_read(), vv_cache_write()
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
|
||||||
header('Content-Type: application/json');
|
|
||||||
header('Cache-Control: no-cache, no-store');
|
|
||||||
require_once dirname(__DIR__) . '/include/monitor.php';
|
|
||||||
|
|
||||||
if (!isset($_GET['live'])) {
|
|
||||||
$cached = vv_cache_read('monitor_remote', 3600);
|
|
||||||
if ($cached !== null) { echo json_encode($cached); exit; }
|
|
||||||
}
|
|
||||||
|
|
||||||
$data = ['remote_hosts' => vv_remote_hosts_stats(), 'ts' => time()];
|
|
||||||
vv_cache_write('monitor_remote', $data);
|
|
||||||
echo json_encode($data);
|
|
||||||
@@ -25,9 +25,18 @@
|
|||||||
// of every orchestrator. That is a distinct intent from conf_toggle.php's commenting
|
// of every orchestrator. That is a distinct intent from conf_toggle.php's commenting
|
||||||
// out — this removes the line, that disables it in place.
|
// out — this removes the line, that disables it in place.
|
||||||
//
|
//
|
||||||
// Indentation is normalised on re-insertion.
|
// Entry text moves verbatim; only a fresh entry is written from the bare path.
|
||||||
// The moved line is rewritten as two spaces and the quoted path, so a script does not
|
// An entry may carry inline arguments ("Media/media_cleaner.sh anime"), a trailing
|
||||||
// carry its old array's formatting into its new one.
|
// comment and its own indentation. Regenerating the line from the script path drops
|
||||||
|
// all three — six live entries in master.conf carry arguments, and stripping them
|
||||||
|
// would leave media_cleaner.sh with no share and fallback.sh without --stop. The
|
||||||
|
// matched line is therefore carried across untouched, which is the same rule
|
||||||
|
// reorderarray.php follows. A script that was in no array is written fresh, indented
|
||||||
|
// to match the entries already in the target.
|
||||||
|
//
|
||||||
|
// One path may hold several entries, and they move together.
|
||||||
|
// "Media/media_cleaner.sh anime" and "… media" are two jobs sharing a path. Every
|
||||||
|
// match is collected and re-inserted, rather than collapsing them into one.
|
||||||
//
|
//
|
||||||
// A move that finds nothing to move still succeeds.
|
// A move that finds nothing to move still succeeds.
|
||||||
// The removal pass is best-effort; only a missing *target* is an error. A script that
|
// The removal pass is best-effort; only a missing *target* is an error. A script that
|
||||||
@@ -115,17 +124,29 @@ if (!$lines) {
|
|||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
$scriptEsc = preg_quote($script, '/');
|
$scriptEsc = preg_quote($script, '/');
|
||||||
$removedLine = null;
|
$removedLines = [];
|
||||||
$inArray = false;
|
$inArray = false;
|
||||||
|
|
||||||
// Step 1: find and remove the script line from whatever array it is currently in.
|
// Step 1: find and remove the script's line(s) from whatever array they are in.
|
||||||
|
//
|
||||||
|
// The original text is carried across verbatim. An entry is not just a path — it may hold
|
||||||
|
// inline arguments ("Media/media_cleaner.sh anime"), a trailing comment, and the file's
|
||||||
|
// indentation, and regenerating the line from the bare path silently drops all three. Six
|
||||||
|
// live entries in master.conf carry arguments; a move that strips them would leave
|
||||||
|
// media_cleaner.sh with no share to clean and fallback.sh without --stop.
|
||||||
|
//
|
||||||
|
// reorderarray.php preserves entry text for exactly this reason. A move must not be the one
|
||||||
|
// operation that loses it.
|
||||||
|
//
|
||||||
|
// A path can legitimately appear more than once in the same array with different arguments,
|
||||||
|
// so every match is collected and re-inserted together rather than collapsing to one.
|
||||||
$newLines = [];
|
$newLines = [];
|
||||||
foreach ($lines as $line) {
|
foreach ($lines as $line) {
|
||||||
if (preg_match('/^\s*[A-Z_]+_SCRIPTS\s*=\s*\(/', $line)) $inArray = true;
|
if (preg_match('/^\s*[A-Z_]+_SCRIPTS\s*=\s*\(/', $line)) $inArray = true;
|
||||||
if ($inArray && preg_match('/^\s*\)\s*(?:#.*)?$/', $line) && !str_contains($line, '(')) $inArray = false;
|
if ($inArray && preg_match('/^\s*\)\s*(?:#.*)?$/', $line) && !str_contains($line, '(')) $inArray = false;
|
||||||
if ($inArray && preg_match('/^\s*(?:#\s*)?"' . $scriptEsc . '(?:\s[^"]*)?"/', $line)) {
|
if ($inArray && preg_match('/^\s*(?:#\s*)?"' . $scriptEsc . '(?:\s[^"]*)?"/', $line)) {
|
||||||
$removedLine = ' "' . $script . '"' . "\n"; // normalise indentation when re-inserting
|
$removedLines[] = $line;
|
||||||
continue; // drop from current location
|
continue; // drop from current location
|
||||||
}
|
}
|
||||||
$newLines[] = $line;
|
$newLines[] = $line;
|
||||||
@@ -136,10 +157,15 @@ if ($toArray) {
|
|||||||
$resultLines = [];
|
$resultLines = [];
|
||||||
$inTarget = false;
|
$inTarget = false;
|
||||||
$inserted = false;
|
$inserted = false;
|
||||||
|
$indent = ' '; // master.conf indents array entries eight spaces
|
||||||
foreach ($newLines as $line) {
|
foreach ($newLines as $line) {
|
||||||
if (preg_match('/^\s*' . preg_quote($toArray, '/') . '\s*=\s*\(/', $line)) $inTarget = true;
|
if (preg_match('/^\s*' . preg_quote($toArray, '/') . '\s*=\s*\(/', $line)) $inTarget = true;
|
||||||
|
// Match the indentation the target array actually uses rather than assuming it.
|
||||||
|
if ($inTarget && preg_match('/^(\s+)(?:#\s*)?"/', $line, $im)) $indent = $im[1];
|
||||||
if ($inTarget && !$inserted && preg_match('/^\s*\)\s*(?:#.*)?$/', $line) && !str_contains($line, '(')) {
|
if ($inTarget && !$inserted && preg_match('/^\s*\)\s*(?:#.*)?$/', $line) && !str_contains($line, '(')) {
|
||||||
$resultLines[] = $removedLine ?? (' "' . $script . '"' . "\n");
|
foreach ($removedLines ?: [$indent . '"' . $script . '"' . "\n"] as $moved) {
|
||||||
|
$resultLines[] = $moved;
|
||||||
|
}
|
||||||
$inTarget = false;
|
$inTarget = false;
|
||||||
$inserted = true;
|
$inserted = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// The Partnership tab's mesh chat: read a channel, post to it, forget a message on this
|
||||||
|
// machine, and mark a channel read.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// The method is the routing. Anything that is not a POST is a read — channel list or one
|
||||||
|
// channel's messages — and every POST carries an action. That keeps the CSRF boundary and the
|
||||||
|
// read/write boundary on the same line, so a mutation cannot arrive un-covered by being
|
||||||
|
// spelled as a GET.
|
||||||
|
//
|
||||||
|
// Reads are computed per request, not cached. Unread counts come from walking the last 200
|
||||||
|
// messages of each channel against this host's read mark; the card polls on a slow cycle and
|
||||||
|
// the store is a small append log, so a cache would add a staleness class for no gain.
|
||||||
|
//
|
||||||
|
// Delivery is not part of the response's success. vv_nc_send() writes locally and spools for
|
||||||
|
// any partner that could not be reached, and the reply carries that spool depth as `queued`.
|
||||||
|
// A sleeping partner is a pending message, not a failed one.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// Read marks are local and per channel. "Unread" is a fact about this operator at this
|
||||||
|
// machine, not something to replicate — the partner has their own idea of what they have seen.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// POST for every mutation, so Unraid's CSRF guard applies. See README-unraid.md.
|
||||||
|
//
|
||||||
|
// Channel ids are resolved against vv_nc_channels() inside the store layer, so a request
|
||||||
|
// cannot name a file this host is not a member of.
|
||||||
|
//
|
||||||
|
// Colour and font are validated to a hex triplet and a three-item list before they are stored,
|
||||||
|
// because they are interpolated into a style attribute when rendered.
|
||||||
|
//
|
||||||
|
// Sending reports the record even when no partner could be reached. The message is written
|
||||||
|
// locally and spooled for retry; saying "failed" over something that is stored and queued
|
||||||
|
// would be the wrong claim.
|
||||||
|
//
|
||||||
|
// REQUEST
|
||||||
|
// GET channels + this host's id + unread counts
|
||||||
|
// GET ?channel=<id> that channel's messages
|
||||||
|
// POST action=send channel=<id> text=… [color=#rrggbb] [font=mono|sans|serif]
|
||||||
|
// POST action=delete channel=<id> id=<msgid> local only
|
||||||
|
// POST action=read channel=<id> mark seen up to now
|
||||||
|
//
|
||||||
|
// RESPONSE
|
||||||
|
// {"ok":true,"me":<host>,"channels":[{…,"unread":N}],"hostnames":{id:name}} channel list
|
||||||
|
// {"ok":true,"me":<host>,"channel":<id>,"messages":[…],"last_read":<ts>} one channel
|
||||||
|
// {"ok":true,"msg":{…},"queued":N} send; N = spooled
|
||||||
|
// {"ok":true|false} delete
|
||||||
|
// {"ok":true} read
|
||||||
|
// {"ok":false,"error":"Unknown channel"|"Nothing to send"|"Could not store message"
|
||||||
|
// |"No message id"|"Unknown action"}
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/node_chat.php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
header('Content-Type: application/json');
|
||||||
|
require_once dirname(__DIR__) . '/include/node_chat.php';
|
||||||
|
|
||||||
|
define('VV_NC_READ_DB', VV_NC_DIR . '/.read.json');
|
||||||
|
|
||||||
|
function vv_nc_read_marks(): array {
|
||||||
|
$j = @json_decode((string)@file_get_contents(VV_NC_READ_DB), true);
|
||||||
|
return is_array($j) ? $j : [];
|
||||||
|
}
|
||||||
|
function vv_nc_set_read(string $ch, int $ts): void {
|
||||||
|
$m = vv_nc_read_marks();
|
||||||
|
$m[$ch] = $ts;
|
||||||
|
@mkdir(VV_NC_DIR, 0755, true);
|
||||||
|
@file_put_contents(VV_NC_READ_DB, json_encode($m), LOCK_EX);
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||||
|
$marks = vv_nc_read_marks();
|
||||||
|
$ch = $_GET['channel'] ?? '';
|
||||||
|
|
||||||
|
if ($ch !== '') {
|
||||||
|
if (!vv_nc_channel($ch)) { echo json_encode(['ok' => false, 'error' => 'Unknown channel']); exit; }
|
||||||
|
echo json_encode([
|
||||||
|
'ok' => true,
|
||||||
|
'me' => vv_nc_me(),
|
||||||
|
'channel' => $ch,
|
||||||
|
'messages' => vv_nc_read($ch, 200),
|
||||||
|
'last_read'=> (int)($marks[$ch] ?? 0),
|
||||||
|
]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$chans = [];
|
||||||
|
foreach (vv_nc_channels() as $c) {
|
||||||
|
$since = (int)($marks[$c['id']] ?? 0);
|
||||||
|
$unread = 0;
|
||||||
|
foreach (vv_nc_read($c['id'], 200) as $m) {
|
||||||
|
if ((int)($m['ts'] ?? 0) > $since && ($m['from'] ?? '') !== vv_nc_me()) $unread++;
|
||||||
|
}
|
||||||
|
$c['unread'] = $unread;
|
||||||
|
$chans[] = $c;
|
||||||
|
}
|
||||||
|
// Hostnames alongside the ids, so the card can show "unRAID-Jayred36" or "HOST2" without a
|
||||||
|
// second request — the mapping is master.conf's and the page should not be guessing it.
|
||||||
|
echo json_encode(['ok' => true, 'me' => vv_nc_me(), 'channels' => $chans,
|
||||||
|
'hostnames' => vv_known_hosts()]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
$action = $_POST['action'] ?? '';
|
||||||
|
$ch = $_POST['channel'] ?? '';
|
||||||
|
if (!vv_nc_channel($ch)) { echo json_encode(['ok' => false, 'error' => 'Unknown channel']); exit; }
|
||||||
|
|
||||||
|
if ($action === 'send') {
|
||||||
|
$text = (string)($_POST['text'] ?? '');
|
||||||
|
if (trim($text) === '') { echo json_encode(['ok' => false, 'error' => 'Nothing to send']); exit; }
|
||||||
|
$msg = vv_nc_send($ch, $text, [
|
||||||
|
'color' => (string)($_POST['color'] ?? ''),
|
||||||
|
'font' => (string)($_POST['font'] ?? ''),
|
||||||
|
'size' => (string)($_POST['size'] ?? ''),
|
||||||
|
'bold' => !empty($_POST['bold']),
|
||||||
|
'italic' => !empty($_POST['italic']),
|
||||||
|
'underline' => !empty($_POST['underline']),
|
||||||
|
]);
|
||||||
|
if (!$msg) { echo json_encode(['ok' => false, 'error' => 'Could not store message']); exit; }
|
||||||
|
// Queued is not failed — say which, so a partner being asleep reads as pending rather than
|
||||||
|
// as an error the operator should act on.
|
||||||
|
$queued = count(glob(VV_NC_SPOOL . '/*/' . $msg['id'] . '.json') ?: []);
|
||||||
|
echo json_encode(['ok' => true, 'msg' => $msg, 'queued' => $queued]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'delete') {
|
||||||
|
$id = preg_replace('/[^a-f0-9]/i', '', (string)($_POST['id'] ?? ''));
|
||||||
|
if ($id === '') { echo json_encode(['ok' => false, 'error' => 'No message id']); exit; }
|
||||||
|
echo json_encode(['ok' => vv_nc_delete_local($ch, $id)]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'read') {
|
||||||
|
vv_nc_set_read($ch, time());
|
||||||
|
echo json_encode(['ok' => true]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||||
@@ -5,6 +5,16 @@
|
|||||||
// reachability, SSH trust, conf-sync state and the shared-service inventory — for the
|
// reachability, SSH trust, conf-sync state and the shared-service inventory — for the
|
||||||
// partnership tab's poll.
|
// partnership tab's poll.
|
||||||
//
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// One call, one document. The endpoint holds no logic and adds no cache of its own — it
|
||||||
|
// encodes whatever vv_partnership_all() returns, so any freshness policy is the library's and
|
||||||
|
// there is only one place it can be changed.
|
||||||
|
//
|
||||||
|
// The whole picture is assembled per request rather than exposed as separate endpoints for
|
||||||
|
// hosts, trust and services. Those answers are read together and judged against each other —
|
||||||
|
// a host that is reachable but has lost SSH trust is a different state from either fact alone,
|
||||||
|
// and splitting them would let the tab render a combination that never existed at one moment.
|
||||||
|
//
|
||||||
// DESIGN PRINCIPLES
|
// DESIGN PRINCIPLES
|
||||||
// Thin transport. Host enumeration, SSH probing and trust evaluation live in
|
// Thin transport. Host enumeration, SSH probing and trust evaluation live in
|
||||||
// include/partnership.php; this file only sets the content type and encodes.
|
// include/partnership.php; this file only sets the content type and encodes.
|
||||||
|
|||||||
@@ -64,9 +64,14 @@ require_once dirname(__DIR__) . '/include/confform.php';
|
|||||||
$files = vv_get_conf_files();
|
$files = vv_get_conf_files();
|
||||||
$out = [];
|
$out = [];
|
||||||
foreach ($files as $f) {
|
foreach ($files as $f) {
|
||||||
|
// "HOST IDENTITIES" is included alongside the partnership subsections because it holds HOST1
|
||||||
|
// and HOST2 — the hostnames the whole partnership is keyed on. They were shown by a separate
|
||||||
|
// "Host Settings" card on the partnership page, which existed only to render those two fields
|
||||||
|
// and is now redundant: one settings surface for the page, not two that both edit master.conf.
|
||||||
$groups = array_values(array_filter(
|
$groups = array_values(array_filter(
|
||||||
vv_conf_all_groups($f),
|
vv_conf_all_groups($f),
|
||||||
fn($g) => stripos($g['subsection'], 'partnership') !== false
|
fn($g) => stripos($g['subsection'], 'partnership') !== false
|
||||||
|
|| stripos($g['subsection'], 'host identities') !== false
|
||||||
));
|
));
|
||||||
if ($groups) {
|
if ($groups) {
|
||||||
$out[] = ['file' => $f, 'groups' => $groups];
|
$out[] = ['file' => $f, 'groups' => $groups];
|
||||||
|
|||||||
@@ -4,6 +4,19 @@
|
|||||||
// Script and document reader. Returns the full text of one .sh or .md file inside
|
// Script and document reader. Returns the full text of one .sh or .md file inside
|
||||||
// SCRIPTS_DIR — the source view behind the scheduler page's script viewer and the docs tab.
|
// SCRIPTS_DIR — the source view behind the scheduler page's script viewer and the docs tab.
|
||||||
//
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Validate, resolve, read. The id is checked against a character class and an extension list
|
||||||
|
// before it is joined to SCRIPTS_DIR, so nothing reaches the filesystem that did not already
|
||||||
|
// look like a repo-relative path.
|
||||||
|
//
|
||||||
|
// The whole file is returned in one response — no ranges, no pagination. These are scripts and
|
||||||
|
// documents, not logs; the largest is a few hundred kilobytes, and a viewer that had to stitch
|
||||||
|
// pages together would be more machinery than the thing it displays.
|
||||||
|
//
|
||||||
|
// Every failure is a JSON body with ok:false, never an HTTP error code. The scheduler's viewer
|
||||||
|
// and the docs tab both parse the response before looking at anything else, so a 404 would
|
||||||
|
// surface as a parse failure rather than as "that file is not there".
|
||||||
|
//
|
||||||
// DESIGN PRINCIPLES
|
// DESIGN PRINCIPLES
|
||||||
// Two extensions, one endpoint.
|
// Two extensions, one endpoint.
|
||||||
// Scripts and their READMEs are read the same way because they are read for the same
|
// Scripts and their READMEs are read the same way because they are read for the same
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ try {
|
|||||||
'dur' => isset($d['end']) ? max(0, (int)$d['end'] - (int)$d['start']) : 0,
|
'dur' => isset($d['end']) ? max(0, (int)$d['end'] - (int)$d['start']) : 0,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
} catch (Exception $e) {}
|
} catch (Exception $e) { vv_log_error('api/recent.php', 'run-log walk failed: ' . $e->getMessage()); }
|
||||||
|
|
||||||
usort($runs, fn($a, $b) => $b['start'] - $a['start']);
|
usort($runs, fn($a, $b) => $b['start'] - $a['start']);
|
||||||
echo json_encode(['ok' => true, 'runs' => array_slice($runs, 0, 24)]);
|
echo json_encode(['ok' => true, 'runs' => array_slice($runs, 0, 24)]);
|
||||||
|
|||||||
@@ -91,6 +91,38 @@ require_once dirname(__DIR__) . '/include/monitor.php';
|
|||||||
|
|
||||||
// ── Live sync log ─────────────────────────────────────────────────────────────
|
// ── Live sync log ─────────────────────────────────────────────────────────────
|
||||||
$action = $_GET['action'] ?? '';
|
$action = $_GET['action'] ?? '';
|
||||||
|
// ── Share list, for the quick-select strip on the window editor ──────────────────────────────
|
||||||
|
// The editor is path-first — type or browse, then Add — which is exact but slow for the case that
|
||||||
|
// is nearly all of them: add one whole share. This answers "what shares exist here", so the strip
|
||||||
|
// can offer them as one click each.
|
||||||
|
//
|
||||||
|
// Read from /boot/config/shares/*.cfg rather than by listing /mnt/user, because a share is a
|
||||||
|
// declared thing: the directory can be absent on a share that has never been written to, and
|
||||||
|
// listing the mount would also invent entries for stray directories that are not shares at all.
|
||||||
|
if ($action === 'shares') {
|
||||||
|
$out = [];
|
||||||
|
foreach (glob('/boot/config/shares/*.cfg') ?: [] as $cfg) {
|
||||||
|
$name = basename($cfg, '.cfg');
|
||||||
|
$raw = (string) @file_get_contents($cfg);
|
||||||
|
$get = function (string $k) use ($raw): string {
|
||||||
|
return preg_match('/^' . $k . '="([^"]*)"/m', $raw, $m) ? $m[1] : '';
|
||||||
|
};
|
||||||
|
$path = '/mnt/user/' . $name;
|
||||||
|
$out[] = [
|
||||||
|
'name' => $name,
|
||||||
|
'path' => $path,
|
||||||
|
'comment' => $get('shareComment'),
|
||||||
|
'pool' => $get('shareCachePool'),
|
||||||
|
'cache' => $get('shareUseCache'),
|
||||||
|
// Present is not the same as declared — a share can exist in conf with no directory yet.
|
||||||
|
'exists' => is_dir($path),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
usort($out, fn($a, $b) => strcasecmp($a['name'], $b['name']));
|
||||||
|
header('Content-Type: application/json');
|
||||||
|
echo json_encode(['ok' => true, 'shares' => $out]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
if ($action === 'rsync_log') {
|
if ($action === 'rsync_log') {
|
||||||
$lockDir = '/tmp/unraid_locks';
|
$lockDir = '/tmp/unraid_locks';
|
||||||
$lines = [];
|
$lines = [];
|
||||||
@@ -150,7 +182,7 @@ $history = [];
|
|||||||
// cleanup, docker updates — and run on their schedule whether or not RSYNC_ENABLED is true. So the
|
// cleanup, docker updates — and run on their schedule whether or not RSYNC_ENABLED is true. So the
|
||||||
// page could report a healthy 56-minute "daily sync" for a subsystem that had not run in a month,
|
// page could report a healthy 56-minute "daily sync" for a subsystem that had not run in a month,
|
||||||
// which is exactly what it was doing: the last entry here is 2026-07-16, the day the global gate
|
// which is exactly what it was doing: the last entry here is 2026-07-16, the day the global gate
|
||||||
// was closed for the HOST2 rebuild.
|
// was closed.
|
||||||
//
|
//
|
||||||
// Deliberately outside the 30-day cutoff. The last transfer is 29 days old as this is written and
|
// Deliberately outside the 30-day cutoff. The last transfer is 29 days old as this is written and
|
||||||
// would have dropped out of the window within days, taking the page from a wrong answer to no
|
// would have dropped out of the window within days, taking the page from a wrong answer to no
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ if ($action === 'list_scripts') {
|
|||||||
$label = str_replace('_', ' ', basename($rel, '.sh'));
|
$label = str_replace('_', ' ', basename($rel, '.sh'));
|
||||||
$groups[$folder][] = ['id' => $rel, 'label' => $label];
|
$groups[$folder][] = ['id' => $rel, 'label' => $label];
|
||||||
}
|
}
|
||||||
} catch (Exception $e) {}
|
} catch (Exception $e) { vv_log_error('api/rsync_win_arrays.php', 'script tree walk failed: ' . $e->getMessage()); }
|
||||||
ksort($groups);
|
ksort($groups);
|
||||||
foreach ($groups as &$g) usort($g, fn($a, $b) => strcmp($a['id'], $b['id']));
|
foreach ($groups as &$g) usort($g, fn($a, $b) => strcmp($a['id'], $b['id']));
|
||||||
echo json_encode(['ok' => true, 'groups' => $groups]);
|
echo json_encode(['ok' => true, 'groups' => $groups]);
|
||||||
|
|||||||
+104
-5
@@ -182,6 +182,18 @@ if ($action === 'ssh_generate') {
|
|||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Where setup hands the operator next, by role.
|
||||||
|
//
|
||||||
|
// Both exits used to be hardcoded to the Scheduler. For HOST1 that is defensible — it has just
|
||||||
|
// written a conf and Scheduler is where you would tune it. For a partner it is a dead end: a
|
||||||
|
// mirror that finishes setup has exactly one job left, installing its key and joining, and that
|
||||||
|
// lives on the Partnership tab. The role was already known here and simply not consulted.
|
||||||
|
function vv_setup_redirect(string $mySlot, string $confFile = 'master.conf'): string {
|
||||||
|
return strtolower($mySlot) === 'host1'
|
||||||
|
? '?tab=scheduler&vv_setup=' . $confFile
|
||||||
|
: '?tab=partnership&vv_setup=' . $confFile;
|
||||||
|
}
|
||||||
|
|
||||||
// ── POST: run conf_populate.sh ─────────────────────────────────────────────────────────────────
|
// ── POST: run conf_populate.sh ─────────────────────────────────────────────────────────────────
|
||||||
if ($action === 'populate') {
|
if ($action === 'populate') {
|
||||||
$script = DEPLOY_DIR . '/conf_populate.sh';
|
$script = DEPLOY_DIR . '/conf_populate.sh';
|
||||||
@@ -195,6 +207,39 @@ if ($action === 'populate') {
|
|||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── POST: defer / undefer a non-blocking checklist item ───────────────────────────────────────
|
||||||
|
// Records the operator's "not now" so the checklist can reach complete without the item being
|
||||||
|
// green. Only the ids api/checklist.php marks deferrable are accepted — a blocking item cannot be
|
||||||
|
// dismissed, because dismissing it would report a mesh as ready when it cannot function.
|
||||||
|
if ($action === 'defer' || $action === 'undefer') {
|
||||||
|
// POST only, and not merely by convention: $action is taken from GET too, and Unraid's CSRF
|
||||||
|
// guard checks POST alone. A GET-reachable mutation here would let any page the operator
|
||||||
|
// visits silently dismiss a checklist item.
|
||||||
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Method not allowed']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$allowed = ['api_key', 'populated', 'emby_key', 'jellyfin_key'];
|
||||||
|
$item = trim($_POST['item'] ?? '');
|
||||||
|
if (!in_array($item, $allowed, true)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Not a deferrable item: ' . $item]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$state = vv_setup_state_read();
|
||||||
|
$key = 'DEFER_' . strtoupper($item);
|
||||||
|
if ($action === 'defer') {
|
||||||
|
$state[$key] = (string)time(); // when, not just whether — a stale decision shows its age
|
||||||
|
} else {
|
||||||
|
unset($state[$key]);
|
||||||
|
}
|
||||||
|
if (!vv_setup_state_write($state)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Failed to write setup state']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
echo json_encode(['ok' => true, 'item' => $item, 'deferred' => $action === 'defer']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
||||||
echo json_encode(['ok' => false, 'error' => 'Method not allowed']);
|
echo json_encode(['ok' => false, 'error' => 'Method not allowed']);
|
||||||
exit;
|
exit;
|
||||||
@@ -234,11 +279,24 @@ if ($action === 'pull') {
|
|||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Resolve HOST1 Tailscale IP
|
// Resolve HOST1's Tailscale IP through the shared resolver, not a bare `tailscale ip -4`.
|
||||||
$ip = trim(shell_exec('tailscale ip -4 ' . escapeshellarg($host1Hostname) . ' 2>/dev/null') ?: '');
|
//
|
||||||
|
// master.conf records the OS hostname (unRAID-Gmer4Lfe). Tailscale knows the same machine by
|
||||||
|
// its own name, and when the two nodes are in different tailnets linked by node sharing, a
|
||||||
|
// shared peer is only addressable by its full name — `unraid-gmer4lfe.tonkinese-monster.ts.net`
|
||||||
|
// resolves while both `unRAID-Gmer4Lfe` and `unraid-gmer4lfe` fall through to public DNS and
|
||||||
|
// fail. A bare lookup therefore reported "is Tailscale running on both servers?" on a mesh
|
||||||
|
// where Tailscale was running perfectly on both.
|
||||||
|
//
|
||||||
|
// vv_resolve_tailscale_ip() already handles this: it tries the direct lookup, then falls back
|
||||||
|
// to an unambiguous prefix match against `tailscale status` with the domain stripped, and
|
||||||
|
// refuses to guess when more than one peer could qualify.
|
||||||
|
$ip = vv_resolve_tailscale_ip($host1Hostname);
|
||||||
if (!$ip) {
|
if (!$ip) {
|
||||||
echo json_encode(['ok' => false, 'error' =>
|
echo json_encode(['ok' => false, 'error' =>
|
||||||
"Cannot resolve Tailscale IP for $host1Hostname — is Tailscale running on both servers?"]);
|
"Cannot resolve a Tailscale address for $host1Hostname. Check `tailscale status` on "
|
||||||
|
. "this server — the name in master.conf must match a peer there, or be an unambiguous "
|
||||||
|
. "prefix of one."]);
|
||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -296,7 +354,7 @@ if ($action === 'pull') {
|
|||||||
|
|
||||||
echo json_encode(['ok' => true, 'host_id' => $hostId, 'conf_file' => $confFile,
|
echo json_encode(['ok' => true, 'host_id' => $hostId, 'conf_file' => $confFile,
|
||||||
'api_key' => $apiKeyResult,
|
'api_key' => $apiKeyResult,
|
||||||
'redirect' => '?tab=scheduler&vv_setup=' . $confFile]);
|
'redirect' => vv_setup_redirect($hostId, $confFile)]);
|
||||||
exit;
|
exit;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -384,6 +442,47 @@ if (!file_exists(CONF_DIR . '/' . $confFile)) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Adopt the owner's custom docker networks ─────────────────────────────────────────────────
|
||||||
|
// Sent by the wizard, which read them out of the owner's host conf that onboard Phase 1 cached
|
||||||
|
// into the RAM conf dir. This is the one setup value a fresh mirror has no way to know: the
|
||||||
|
// template ships NETWORK_CONNECT_NETWORKS with its only entry commented out, and the owner then
|
||||||
|
// deploys containers here onto a network named in the *owner's* templates. An empty list is what
|
||||||
|
// left twelve containers created against a network that did not exist.
|
||||||
|
//
|
||||||
|
// Outside the create block above, so it applies to a conf that already exists — the wizard is
|
||||||
|
// re-runnable and a mirror rebuilt against an existing host conf needs this just as much.
|
||||||
|
//
|
||||||
|
// Merged, never replaced: anything already listed here was put there deliberately.
|
||||||
|
$netsRaw = trim((string)($_POST['networks'] ?? ''));
|
||||||
|
if ($netsRaw !== '' && $hostIdLow !== 'host1') {
|
||||||
|
// Written into a file that bash sources, so the name is validated rather than trusted.
|
||||||
|
// Docker's own charset for a network name is a superset of this; anything outside it is
|
||||||
|
// far more likely to be an injection attempt than a real network.
|
||||||
|
$nets = array_values(array_filter(
|
||||||
|
array_map('trim', explode(',', $netsRaw)),
|
||||||
|
fn($n) => $n !== ''
|
||||||
|
&& preg_match('/^[A-Za-z0-9][A-Za-z0-9._-]{0,62}$/', $n)
|
||||||
|
&& !preg_match('/^(bridge|host|none|br\d)/i', $n)
|
||||||
|
));
|
||||||
|
if ($nets) {
|
||||||
|
$netVar = $hostId . '_NETWORK_CONNECT_NETWORKS';
|
||||||
|
vv_conf_edit($confFile, function (string $cur) use ($netVar, $nets): ?string {
|
||||||
|
$existing = vv_parse_conf_list($cur, $netVar);
|
||||||
|
$merged = $existing;
|
||||||
|
foreach ($nets as $n) {
|
||||||
|
if (!in_array($n, $merged, true)) $merged[] = $n;
|
||||||
|
}
|
||||||
|
if ($merged === $existing) return null; // already adopted — no write, no audit noise
|
||||||
|
$body = '';
|
||||||
|
foreach ($merged as $n) $body .= ' "' . $n . '"' . "\n";
|
||||||
|
$new = preg_replace(
|
||||||
|
'/^(\s*' . preg_quote($netVar, '/') . '\s*=\s*\()(.*?)(^\s*\))/ms',
|
||||||
|
"\$1\n" . $body . '$3', $cur, 1, $count);
|
||||||
|
return ($count === 1 && $new !== null) ? $new : null;
|
||||||
|
}, [], [$netVar]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Write setup state file — lets partner servers know HOST1 is configured.
|
// Write setup state file — lets partner servers know HOST1 is configured.
|
||||||
// Read-modify-write: vv_setup_state_write() replaces the file wholesale, and re-running the
|
// Read-modify-write: vv_setup_state_write() replaces the file wholesale, and re-running the
|
||||||
// wizard must not erase onboarding progress recorded by the partnership phases.
|
// wizard must not erase onboarding progress recorded by the partnership phases.
|
||||||
@@ -408,5 +507,5 @@ echo json_encode([
|
|||||||
'api_key' => $apiKeyResult,
|
'api_key' => $apiKeyResult,
|
||||||
'needs_migration'=> $needsMigration,
|
'needs_migration'=> $needsMigration,
|
||||||
'migrate_to' => $needsMigration ? ($storageInternal === 'true' ? 'internal' : 'flash') : null,
|
'migrate_to' => $needsMigration ? ($storageInternal === 'true' ? 'internal' : 'flash') : null,
|
||||||
'redirect' => '?tab=scheduler&vv_setup=master.conf',
|
'redirect' => vv_setup_redirect($mySlot),
|
||||||
]);
|
]);
|
||||||
|
|||||||
@@ -53,6 +53,40 @@
|
|||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
require_once dirname(__DIR__) . '/include/scheduler.php';
|
require_once dirname(__DIR__) . '/include/scheduler.php';
|
||||||
|
|
||||||
|
// ?id=<Category/name.sh> — one job, including jobs that are not on the schedule.
|
||||||
|
//
|
||||||
|
// The loop below only knows about scheduled jobs, so a job like Partnership/partnership_onboard.sh
|
||||||
|
// — launched on demand, never cronned — was invisible to every status caller even though
|
||||||
|
// run_job.sh writes it a stat file like any other. The UI had nothing to poll, which is why
|
||||||
|
// pressing Onboard produced no visible change for the minutes it then ran.
|
||||||
|
if (isset($_GET['id'])) {
|
||||||
|
$id = trim($_GET['id']);
|
||||||
|
if (!preg_match('/^[a-zA-Z0-9_.\/\-]+\.sh$/', $id) || str_contains($id, '..')) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'Invalid job id']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$statFile = vv_job_stat_path($id);
|
||||||
|
if (!file_exists($statFile)) {
|
||||||
|
echo json_encode(['ok' => true, 'id' => $id, 'status' => 'never_run']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$stat = json_decode(@file_get_contents($statFile) ?: '{}', true) ?: [];
|
||||||
|
$status = $stat['status'] ?? 'unknown';
|
||||||
|
// Same liveness rule as the loop: a dead runner reports error, never "still running".
|
||||||
|
if ($status === 'running' && !empty($stat['pid']) && !file_exists("/proc/{$stat['pid']}")) {
|
||||||
|
$status = 'error';
|
||||||
|
}
|
||||||
|
echo json_encode([
|
||||||
|
'ok' => true,
|
||||||
|
'id' => $id,
|
||||||
|
'status' => $status,
|
||||||
|
'start' => $stat['start'] ?? null,
|
||||||
|
'end' => $stat['end'] ?? null,
|
||||||
|
'exit' => $stat['exit'] ?? null,
|
||||||
|
]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
$result = [];
|
$result = [];
|
||||||
foreach (array_keys(vv_schedule_load()) as $id) {
|
foreach (array_keys(vv_schedule_load()) as $id) {
|
||||||
$statFile = vv_job_stat_path($id);
|
$statFile = vv_job_stat_path($id);
|
||||||
|
|||||||
@@ -5,6 +5,16 @@
|
|||||||
// resource, docker, system, storage, network and stability — together with the thresholds
|
// resource, docker, system, storage, network and stability — together with the thresholds
|
||||||
// each one is judging against, for the watchdog tab's poll.
|
// each one is judging against, for the watchdog tab's poll.
|
||||||
//
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Served from a 5-minute cache unless ?live is present. Assembling this payload reads every
|
||||||
|
// watchdog's state files and resolves every threshold out of master.conf, which is far more
|
||||||
|
// work than the tab's poll needs — the watchdogs themselves only run every 15 minutes, so a
|
||||||
|
// fresher answer would describe the same cycle.
|
||||||
|
//
|
||||||
|
// The cache is consulted before include/watchdog.php is even loaded, so a cache hit costs one
|
||||||
|
// file read and nothing else. ?live skips the read, recomputes, and writes the result back, so
|
||||||
|
// an explicit refresh also benefits the next visitor rather than being discarded.
|
||||||
|
//
|
||||||
// DESIGN PRINCIPLES
|
// DESIGN PRINCIPLES
|
||||||
// Thin transport. State-file parsing and threshold resolution live in
|
// Thin transport. State-file parsing and threshold resolution live in
|
||||||
// include/watchdog.php; this file only sets the content type and encodes.
|
// include/watchdog.php; this file only sets the content type and encodes.
|
||||||
@@ -15,7 +25,9 @@
|
|||||||
// against, so the page never has to fetch the two independently and risk mismatching
|
// against, so the page never has to fetch the two independently and risk mismatching
|
||||||
// them across a conf edit.
|
// them across a conf edit.
|
||||||
//
|
//
|
||||||
// No parameters. Which watchdogs exist is fixed by the codebase, not by the request.
|
// The only parameter is a freshness override. ?live decides how old an answer may be, never
|
||||||
|
// what is in it — which watchdogs exist is fixed by the codebase, and no request can select,
|
||||||
|
// filter or widen the set.
|
||||||
//
|
//
|
||||||
// OPERATIONAL SAFEGUARDS
|
// OPERATIONAL SAFEGUARDS
|
||||||
// Read-only. Nothing here clears a strike, lifts a skip-list entry, restarts a container,
|
// Read-only. Nothing here clears a strike, lifts a skip-list entry, restarts a container,
|
||||||
@@ -35,7 +47,7 @@
|
|||||||
//
|
//
|
||||||
// Served from cache, collected only on a miss.
|
// Served from cache, collected only on a miss.
|
||||||
// vv_wd_all() SSHes to every configured partner, so its cost is set by the slowest node
|
// vv_wd_all() SSHes to every configured partner, so its cost is set by the slowest node
|
||||||
// rather than by how much data there is. Measured at 8.3s on this host with HOST2 down —
|
// rather than by how much data there is. Measured at 8.3s on this host with the partner unreachable —
|
||||||
// paid by every visitor, every 30 seconds, because the tab polls. The cache check happens
|
// paid by every visitor, every 30 seconds, because the tab polls. The cache check happens
|
||||||
// before the heavy include, so a hit costs one file read and no SSH at all.
|
// before the heavy include, so a hit costs one file read and no SSH at all.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -95,7 +95,7 @@
|
|||||||
//
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/config.php vv_conf_vars(), SCRIPTS_DIR
|
// include/config.php vv_conf_vars(), SCRIPTS_DIR
|
||||||
// Media/upgrade_webhook_handler.sh the backgrounded handler
|
// Arrs_Stack/upgrade_webhook_handler.sh the backgrounded handler
|
||||||
// master.conf DOWNLOAD_WEBHOOK_ENABLED
|
// master.conf DOWNLOAD_WEBHOOK_ENABLED
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
|
|||||||
+121
-36
@@ -190,7 +190,32 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
.vv-nb-settings { display: flex; align-items: center; gap: 8px; flex-wrap: wrap;
|
.vv-nb-settings { display: flex; align-items: center; gap: 8px; flex-wrap: wrap;
|
||||||
padding: 5px 10px; background: #101010; border-bottom: 1px solid #1a1a1a; }
|
padding: 5px 10px; background: #101010; border-bottom: 1px solid #1a1a1a; }
|
||||||
|
|
||||||
/* ── Monitor: locked row height + scrollable cards ────────────────────────── */
|
/* ── Monitor: the board ───────────────────────────────────────────────────────
|
||||||
|
Only the half of the layout that is the same at every width lives here. Which cards exist,
|
||||||
|
how wide each one is, what order they sit in, how many columns the board has at this width
|
||||||
|
and how tall a row may be are all generated by include/monitor_board.php and arrive as
|
||||||
|
--vv-cols, --vv-sp and --vv-rowdiv. Do not put a column count or a breakpoint in this file;
|
||||||
|
they are arithmetic over the card floor, and that arithmetic has one home.
|
||||||
|
|
||||||
|
minmax(0, 1fr), not 1fr: bare 1fr means minmax(auto, 1fr), and that auto floor resolves to the
|
||||||
|
larger of min-content and min-width — one card holding an unbreakable string would push its
|
||||||
|
column past its share and drag the grid out of the container. The row axis carried this guard
|
||||||
|
for years; the column axis never did.
|
||||||
|
|
||||||
|
min-width: 0 on the cards for the same reason, overriding the 200px .vv-card floor the flex
|
||||||
|
layouts on other pages need. On this board the ladder is what guarantees a card's width, so a
|
||||||
|
card defending its own floor can only overflow the track the ladder just sized for it. */
|
||||||
|
#vv-monitor {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(var(--vv-cols, 8), minmax(0, 1fr));
|
||||||
|
gap: 12px;
|
||||||
|
width: 100%;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
#vv-monitor > .vv-card {
|
||||||
|
grid-column: span var(--vv-sp, 1);
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* Cards on the monitor grid are flex columns — h3 pins, body scrolls */
|
/* Cards on the monitor grid are flex columns — h3 pins, body scrolls */
|
||||||
#vv-monitor .vv-card {
|
#vv-monitor .vv-card {
|
||||||
@@ -210,32 +235,65 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
/* Hide scrollbars on any nested scrollable div inside monitor cards */
|
/* Hide scrollbars on any nested scrollable div inside monitor cards */
|
||||||
#vv-monitor .vv-card div::-webkit-scrollbar { display: none; }
|
#vv-monitor .vv-card div::-webkit-scrollbar { display: none; }
|
||||||
|
|
||||||
|
/* System card, held to the Network card's height.
|
||||||
|
At eight columns these two share a row (system 1 + ups 1 + cpu 2 + memory 2 + network 2), so
|
||||||
|
the grid already stretches them to the same height and nothing here is needed. Below eight the
|
||||||
|
board re-cuts into rows of four: System lands in row 1 with UPS and CPU, Network in row 2 with
|
||||||
|
Memory, and once rows are sized to content (vv_mon_rung_overflows) the two are free to differ.
|
||||||
|
System is the taller — it carries a host block and a Varaverk block — so it is the one capped.
|
||||||
|
|
||||||
|
222px is Network measured rather than guessed: 2 border + 24 padding + 25 h3 + 61 header block
|
||||||
|
+ 110 canvas. The canvas is a fixed 110px, which is what makes the number worth writing down —
|
||||||
|
most of Network's height cannot drift. What can is the IP list (LAN/EXT/TS), roughly 18px a row,
|
||||||
|
so a host resolving fewer of the three leaves System slightly tall against it.
|
||||||
|
|
||||||
|
Two ids on the selectors, to outrank #vv-monitor .vv-card from the auto-rows hatch above. That
|
||||||
|
hatch lifts overflow off every card so content-sized rows can work; this card is the one place
|
||||||
|
the clamps have to go back on, or a max-height with overflow:visible would draw straight
|
||||||
|
through the card's own border. */
|
||||||
|
@media (max-width: 1383px), (max-height: 700px) {
|
||||||
|
#vv-monitor #vv-system { max-height: 222px; overflow: hidden; }
|
||||||
|
#vv-monitor #vv-system > div { overflow-y: auto; min-height: 0; }
|
||||||
|
}
|
||||||
|
|
||||||
/* Dynamic row heights capped per screen tier — rows size to content, never exceed the cap.
|
/* Dynamic row heights capped per screen tier — rows size to content, never exceed the cap.
|
||||||
minmax(0, Xpx): track is content-driven but capped; align-items:stretch makes all cards
|
minmax(0, Xpx): track is content-driven but capped; align-items:stretch makes all cards
|
||||||
in a row fill the track, so short cards (Pools, Watchdog) match tall ones (Array).
|
in a row fill the track, so short cards (Pools, Watchdog) match tall ones (Array).
|
||||||
Breakpoints are viewport height (after browser chrome), not screen height. */
|
Breakpoints are viewport height (after browser chrome), not screen height.
|
||||||
|
|
||||||
|
The divisor is --vv-rowdiv, not a literal 4. It used to be 4 because the board happened to be
|
||||||
|
four rows at eight columns, so the two numbers were the same by accident and only one of them
|
||||||
|
moved when the column count changed. monitor_board.php derives it as min(rows at this rung,
|
||||||
|
VV_MON_ROWS_PER_SCREEN): the cap means "no card taller than this fraction of the screen", and a
|
||||||
|
board short enough to fit divides by its own row count and fills the screen instead. */
|
||||||
|
|
||||||
/* ~720p (viewport ≤ 700px) */
|
|
||||||
@media (min-width: 481px) and (max-height: 700px) {
|
|
||||||
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 160px) / 4)); }
|
|
||||||
}
|
|
||||||
/* ~1080p (viewport 701–1100px) */
|
/* ~1080p (viewport 701–1100px) */
|
||||||
@media (min-width: 481px) and (min-height: 701px) and (max-height: 1100px) {
|
@media (min-height: 701px) and (max-height: 1100px) {
|
||||||
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 240px) / 4)); }
|
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 240px) / var(--vv-rowdiv, 4))); }
|
||||||
}
|
}
|
||||||
/* ~1440p (viewport 1101–1450px) — calibrated on 15" 1440p display */
|
/* ~1440p (viewport 1101–1450px) — calibrated on 15" 1440p display */
|
||||||
@media (min-width: 481px) and (min-height: 1101px) and (max-height: 1450px) {
|
@media (min-height: 1101px) and (max-height: 1450px) {
|
||||||
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 335px) / 4)); }
|
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 335px) / var(--vv-rowdiv, 4))); }
|
||||||
}
|
}
|
||||||
/* ~4K (viewport > 1450px) */
|
/* ~4K (viewport > 1450px) */
|
||||||
@media (min-width: 481px) and (min-height: 1451px) {
|
@media (min-height: 1451px) {
|
||||||
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 500px) / 4)); }
|
#vv-monitor { grid-auto-rows: minmax(0, calc((100vh - 500px) / var(--vv-rowdiv, 4))); }
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Mobile: natural heights, let page scroll */
|
/* Natural heights, page scrolls. Height is what makes a phone in landscape unusable, not width:
|
||||||
@media (max-width: 480px) {
|
this hatch was keyed to max-width:480 alone, so portrait (393px wide) got content-sized rows and
|
||||||
#vv-monitor { grid-auto-rows: auto !important; }
|
worked, while landscape (851x393) missed it and inherited the four-row cap — 58px cards with
|
||||||
#vv-monitor .vv-card { overflow: visible !important; }
|
overflow:hidden and scrollbars disabled. Nothing was visible and nothing said so.
|
||||||
|
|
||||||
|
Height only now. The width half was a literal 695 that had to be kept in step with
|
||||||
|
VV_MON_CARD_FLOOR by hand; monitor_board.php emits these same three rules for every rung whose
|
||||||
|
row count exceeds VV_MON_ROWS_PER_SCREEN — see vv_mon_rung_overflows() — which covers that band
|
||||||
|
and the four-column one above it from the arithmetic instead. What is left here is the case the
|
||||||
|
ladder cannot see: a window wide enough for a rung that does fit a screen, on a screen too short
|
||||||
|
to give those rows a usable height. */
|
||||||
|
@media (max-height: 700px) {
|
||||||
|
#vv-monitor { grid-auto-rows: auto; }
|
||||||
|
#vv-monitor .vv-card { overflow: visible; }
|
||||||
#vv-monitor .vv-card > div { overflow-y: visible; min-height: auto; }
|
#vv-monitor .vv-card > div { overflow-y: visible; min-height: auto; }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,13 +309,17 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
default stays start so a card added later has to say so. */
|
default stays start so a card added later has to say so. */
|
||||||
#vv-monitor-ai {
|
#vv-monitor-ai {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: repeat(8, 1fr);
|
grid-template-columns: repeat(8, minmax(0, 1fr));
|
||||||
gap: 12px;
|
gap: 12px;
|
||||||
margin-top: 12px;
|
margin-top: 12px;
|
||||||
width: 100%;
|
width: 100%;
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
align-items: start;
|
align-items: start;
|
||||||
}
|
}
|
||||||
|
/* Same reason as the board above: the ladder guarantees the width, so a card defending the 200px
|
||||||
|
.vv-card floor can only overflow the track it was given. At the four-column rung's own minimum
|
||||||
|
(696px) a column is 160px, and that floor would have pushed this row out of the page. */
|
||||||
|
#vv-monitor-ai > .vv-card { min-width: 0; }
|
||||||
|
|
||||||
/* Explicit placement, two rows. The left column carries AI over Tokens; Conversations and the
|
/* Explicit placement, two rows. The left column carries AI over Tokens; Conversations and the
|
||||||
Assistant span both rows, so the Assistant sets the height and the two stacked cards divide
|
Assistant span both rows, so the Assistant sets the height and the two stacked cards divide
|
||||||
@@ -354,8 +416,14 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
#vv-ai-tokens-card .vv-ai-tok-head:first-child { margin-top: 0; }
|
#vv-ai-tokens-card .vv-ai-tok-head:first-child { margin-top: 0; }
|
||||||
|
|
||||||
/* Medium width — 4 columns, mirroring #vv-monitor's own breakpoint. The 1/2/5 spans do not
|
/* Medium width — 4 columns, mirroring #vv-monitor's own breakpoint. The 1/2/5 spans do not
|
||||||
survive the narrower grid: 5 of 4 would silently overflow the track. */
|
survive the narrower grid: 5 of 4 would silently overflow the track.
|
||||||
@media (max-width: 1024px) {
|
|
||||||
|
1383 and 695 below are the four- and two-column rungs from include/monitor_board.php, minus
|
||||||
|
one. This row keeps its own placement — the 1/2/5 split is not a power of two and does not want
|
||||||
|
to be — but it cannot keep its own breakpoints: at any width where these two disagree the board
|
||||||
|
above is four columns while this row is still eight, and the page visibly stops being one
|
||||||
|
board. If VV_MON_CARD_FLOOR moves, these move with it. */
|
||||||
|
@media (max-width: 1383px) {
|
||||||
#vv-monitor-ai { grid-template-columns: repeat(4, 1fr) !important; }
|
#vv-monitor-ai { grid-template-columns: repeat(4, 1fr) !important; }
|
||||||
#vv-ai-stats-card { grid-column: 1 / span 1 !important; grid-row: 1 !important; }
|
#vv-ai-stats-card { grid-column: 1 / span 1 !important; grid-row: 1 !important; }
|
||||||
#vv-ai-tokens-card { grid-column: 1 / span 1 !important; grid-row: 2 !important; }
|
#vv-ai-tokens-card { grid-column: 1 / span 1 !important; grid-row: 2 !important; }
|
||||||
@@ -365,10 +433,14 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
#vv-ai-assistant-card { grid-column: 1 / span 4 !important; grid-row: 3 !important; }
|
#vv-ai-assistant-card { grid-column: 1 / span 4 !important; grid-row: 3 !important; }
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Phone — single column, natural heights, page scrolls. Matching heights is meaningless once
|
/* Narrow — single column, natural heights, page scrolls. Matching heights is meaningless once
|
||||||
the cards are stacked, and the inner scroll has to go with it: on a page that already scrolls,
|
the cards are stacked, and the inner scroll has to go with it: on a page that already scrolls,
|
||||||
a scroll region inside it is a trap for a thumb. */
|
a scroll region inside it is a trap for a thumb.
|
||||||
@media (max-width: 480px) {
|
|
||||||
|
695, where the board above drops to two columns, rather than the 480 this used to use. There is
|
||||||
|
no two-column layout for this row and there should not be: its 1/1/3/4 split at four columns is
|
||||||
|
already the narrowest arrangement in which the Assistant transcript is readable. */
|
||||||
|
@media (max-width: 695px) {
|
||||||
#vv-monitor-ai { grid-template-columns: 1fr !important; }
|
#vv-monitor-ai { grid-template-columns: 1fr !important; }
|
||||||
/* grid-row has to be released along with grid-column. The explicit rows above are the whole
|
/* grid-row has to be released along with grid-column. The explicit rows above are the whole
|
||||||
reason Tokens sits under AI, and left in place on a single-column grid they would stack
|
reason Tokens sits under AI, and left in place on a single-column grid they would stack
|
||||||
@@ -389,14 +461,14 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
.vv-cpu-core { min-width: 7px !important; }
|
.vv-cpu-core { min-width: 7px !important; }
|
||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 1024px) {
|
/* The four-column rung. Nothing about placement is here any more: the column count and every
|
||||||
#vv-monitor { grid-template-columns: repeat(4, 1fr) !important; }
|
span come from the generated ladder, and the six !important overrides this block used to carry
|
||||||
#vv-docker { grid-column: span 4 !important; }
|
existed only to out-shout the inline styles the cards no longer have. Two of them
|
||||||
/* Reset explicit placements so cards reflow in the 4-col grid */
|
(#vv-storage-card, #vv-array-card) were resetting hand-placed column indexes — 3/span 2 and
|
||||||
#vv-docker-folders { grid-column: span 4 !important; }
|
5/span 4 — which is the pair that made the grid invent implicit columns at any count but eight.
|
||||||
#vv-parity-card { grid-column: auto !important; }
|
What is left is the one thing that genuinely belongs to width: at this rung the Streams card is
|
||||||
#vv-storage-card { grid-column: auto !important; }
|
half the board rather than a quarter, and its right-hand chips no longer fit. */
|
||||||
#vv-array-card { grid-column: auto !important; }
|
@media (max-width: 1383px) {
|
||||||
/* Streams header: hide right chip group entirely, keep server badges + media type */
|
/* Streams header: hide right chip group entirely, keep server badges + media type */
|
||||||
.vv-stream-right { display: none; }
|
.vv-stream-right { display: none; }
|
||||||
}
|
}
|
||||||
@@ -441,10 +513,7 @@ body.vv-fullscreen #displaybox { padding-left: 1rem !important; padding-top: .5r
|
|||||||
.vv-cpu-cores { gap: 2px !important; }
|
.vv-cpu-cores { gap: 2px !important; }
|
||||||
.vv-cpu-core { min-width: 6px !important; }
|
.vv-cpu-core { min-width: 6px !important; }
|
||||||
|
|
||||||
/* Monitor single-column — explicit placement cards need override too */
|
/* The single-column rung is generated — see include/monitor_board.php. */
|
||||||
#vv-monitor { grid-template-columns: 1fr !important; }
|
|
||||||
#vv-monitor > .vv-card { grid-column: 1 / -1 !important; }
|
|
||||||
#vv-docker-folders { grid-column: 1 / -1 !important; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Shared footer (Save Schedule left, info right) — same min-height so log card ends level with script cards */
|
/* Shared footer (Save Schedule left, info right) — same min-height so log card ends level with script cards */
|
||||||
@@ -1167,6 +1236,19 @@ mark { background: #5d4037; color: #ffcc80; border-radius: 2px; }
|
|||||||
.vv-chip-group-device, .vv-chip-group-res, .vv-chip-group-codec { display: contents; }
|
.vv-chip-group-device, .vv-chip-group-res, .vv-chip-group-codec { display: contents; }
|
||||||
.vv-stream-empty { color: #555; font-style: italic; font-size: 12px; margin: 4px 0; }
|
.vv-stream-empty { color: #555; font-style: italic; font-size: 12px; margin: 4px 0; }
|
||||||
.vv-stream-empty span { font-size: 11px; color: #444; }
|
.vv-stream-empty span { font-size: 11px; color: #444; }
|
||||||
|
|
||||||
|
/* Streams scope control. Deliberately named vv-strm-* rather than reusing a bare utility name:
|
||||||
|
Unraid Connect injects a global Tailwind layer into every page, and a class named after a
|
||||||
|
utility gets whatever that layer says. */
|
||||||
|
.vv-strm-tab { background: #111; border: 1px solid #242424; color: #555; font-size: 10px;
|
||||||
|
padding: 2px 10px; border-radius: 3px; cursor: pointer; text-transform: uppercase;
|
||||||
|
letter-spacing: .05em; }
|
||||||
|
.vv-strm-tab:hover { color: #bbb; border-color: #3a3a3a; }
|
||||||
|
.vv-strm-tab.on { color: #4caf50; border-color: #2d4a2d; background: #0d1a0d; }
|
||||||
|
/* The node a stream is playing on. Only rendered in mesh scope — in local scope every row is
|
||||||
|
this host and the label would be noise on every line. */
|
||||||
|
.vv-strm-node { font-size: 9px; color: #4a7a9f; border: 1px solid #24384a; background: #0d151c;
|
||||||
|
border-radius: 2px; padding: 0 4px; margin-left: 5px; white-space: nowrap; }
|
||||||
.vv-stream-row { margin-bottom: 10px; padding-bottom: 10px; border-bottom: 1px solid #282828; }
|
.vv-stream-row { margin-bottom: 10px; padding-bottom: 10px; border-bottom: 1px solid #282828; }
|
||||||
.vv-stream-row:last-child { margin-bottom: 0; padding-bottom: 0; border-bottom: none; }
|
.vv-stream-row:last-child { margin-bottom: 0; padding-bottom: 0; border-bottom: none; }
|
||||||
.vv-stream-top { display: flex; align-items: center; gap: 6px; margin-bottom: 4px; }
|
.vv-stream-top { display: flex; align-items: center; gap: 6px; margin-bottom: 4px; }
|
||||||
@@ -1283,8 +1365,11 @@ code.vv-unknown-var { color: #ff9800; background: #1f130d; }
|
|||||||
.vv-df-fname { flex: 1; font-size: 12px; color: #aaa; font-weight: 500;
|
.vv-df-fname { flex: 1; font-size: 12px; color: #aaa; font-weight: 500;
|
||||||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||||
.vv-df-folder-body { padding-left: 10px; padding-bottom: 3px; }
|
.vv-df-folder-body { padding-left: 10px; padding-bottom: 3px; }
|
||||||
.vv-df-container { display: flex; align-items: center; gap: 7px; padding: 3px 6px;
|
/* padding-left is 4px against a 2px border so a container with no fallback tier still lines up
|
||||||
cursor: pointer; border-radius: 3px; user-select: none; }
|
with one that has a stripe — the border is always present, only its colour changes. */
|
||||||
|
.vv-df-container { display: flex; align-items: center; gap: 7px; padding: 3px 6px 3px 4px;
|
||||||
|
cursor: pointer; border-radius: 3px; user-select: none;
|
||||||
|
border-left: 2px solid transparent; }
|
||||||
.vv-df-container:hover { background: rgba(255,255,255,0.04); }
|
.vv-df-container:hover { background: rgba(255,255,255,0.04); }
|
||||||
.vv-df-active { background: rgba(100,149,237,0.1) !important;
|
.vv-df-active { background: rgba(100,149,237,0.1) !important;
|
||||||
outline: 1px solid rgba(100,149,237,0.35);
|
outline: 1px solid rgba(100,149,237,0.35);
|
||||||
|
|||||||
@@ -547,7 +547,64 @@ function vv_ai_recent_logs(int $max = 40): array {
|
|||||||
return $out;
|
return $out;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The figures every AI surface draws: the banner, the Scheduler dock and the Monitor card.
|
||||||
|
//
|
||||||
|
// On a mirror these describe the owner, because on a mirror there is nothing local to describe.
|
||||||
|
// Collected locally it probed an Ollama that is not installed and an index that is never built,
|
||||||
|
// and the Monitor card said "Ollama unreachable / Index not built" — both true of that box and
|
||||||
|
// both the wrong question, since the model it actually uses was up the whole time.
|
||||||
|
//
|
||||||
|
// Routed here rather than in the cache writer so that every consumer follows. api/ai.php already
|
||||||
|
// sends the stats *action* to the owner, so the tab was right while the card was wrong; this is
|
||||||
|
// what makes the two agree.
|
||||||
function vv_ai_stats(): array {
|
function vv_ai_stats(): array {
|
||||||
|
return vv_ai_is_owner() ? vv_ai_stats_local() : vv_ai_stats_shared();
|
||||||
|
}
|
||||||
|
|
||||||
|
// What the owner reports about itself, fetched once and cached by the caller. A transport failure
|
||||||
|
// is reported as a failed health check rather than as absent hardware: "cannot reach the owner"
|
||||||
|
// and "the owner has no model" are different faults with different fixes, and the local-collection
|
||||||
|
// version could only ever express the second one.
|
||||||
|
function vv_ai_stats_shared(): array {
|
||||||
|
require_once __DIR__ . '/ai_rpc.php';
|
||||||
|
|
||||||
|
$status = 200;
|
||||||
|
$owner = vv_ai_owner_host();
|
||||||
|
$name = trim((string)(vv_conf_vars()[strtoupper($owner)] ?? $owner));
|
||||||
|
$r = vv_ai_rpc('stats', [], false, $status);
|
||||||
|
|
||||||
|
if (($r['ok'] ?? false) && is_array($r['stats'] ?? null)) {
|
||||||
|
$stats = $r['stats'];
|
||||||
|
$stats['served_by'] = $name; // so a surface can say whose numbers these are
|
||||||
|
return $stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
$why = (string)($r['error'] ?? 'no response from the AI owner');
|
||||||
|
return [
|
||||||
|
'enabled' => vv_ai_enabled(),
|
||||||
|
'model' => '',
|
||||||
|
'embed_model' => '',
|
||||||
|
'url' => '',
|
||||||
|
'k' => 0,
|
||||||
|
'served_by' => $name,
|
||||||
|
// Shapes kept identical to the local collection. The Monitor card reads runtime.reachable
|
||||||
|
// and index.exists directly, and a missing key there renders as a broken card rather than
|
||||||
|
// as the outage it is.
|
||||||
|
'index' => ['exists' => false, 'chunks' => 0, 'stale' => false],
|
||||||
|
'runtime' => ['reachable' => false, 'loaded' => false, 'offload_pct' => null],
|
||||||
|
'loaded' => null,
|
||||||
|
'health' => [[
|
||||||
|
'id' => 'owner',
|
||||||
|
'label' => 'AI owner',
|
||||||
|
'state' => 'bad',
|
||||||
|
'detail' => $why,
|
||||||
|
'fix' => "Check that $name is up and that this node's SSH key still reaches it",
|
||||||
|
]],
|
||||||
|
'ts' => time(),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
function vv_ai_stats_local(): array {
|
||||||
$cfg = vv_ai_config();
|
$cfg = vv_ai_config();
|
||||||
return [
|
return [
|
||||||
'enabled' => $cfg['enabled'],
|
'enabled' => $cfg['enabled'],
|
||||||
@@ -596,6 +653,10 @@ function vv_ai_stats_cached(bool $live = false): array {
|
|||||||
function vv_ai_monitor_block(array $stats): array {
|
function vv_ai_monitor_block(array $stats): array {
|
||||||
return [
|
return [
|
||||||
'model' => $stats['model'] ?? '',
|
'model' => $stats['model'] ?? '',
|
||||||
|
// Empty on the owner, the owner's hostname on a mirror. The card draws a model and an
|
||||||
|
// index that live on another machine, and a dashboard that shows one node's figures under
|
||||||
|
// another node's heading is the kind of quiet wrongness this project keeps finding.
|
||||||
|
'served_by' => $stats['served_by'] ?? '',
|
||||||
'runtime' => $stats['runtime'] ?? [],
|
'runtime' => $stats['runtime'] ?? [],
|
||||||
'index' => $stats['index'] ?? [],
|
'index' => $stats['index'] ?? [],
|
||||||
];
|
];
|
||||||
@@ -872,16 +933,26 @@ function vv_ai_token_retain(): int {
|
|||||||
|
|
||||||
// LOCK_EX because two turns can finish together — the composer allows a second question while
|
// LOCK_EX because two turns can finish together — the composer allows a second question while
|
||||||
// the first is still generating, and each runs in its own detached worker.
|
// the first is still generating, and each runs in its own detached worker.
|
||||||
function vv_ai_token_record(string $profile, string $source, int $prompt, int $completion, ?float $tokS): void {
|
//
|
||||||
|
// $node is who *asked*, which stopped being the same as who generated when the mesh began sharing
|
||||||
|
// one AI: a turn started on a mirror runs on the owner and would otherwise be billed to the owner,
|
||||||
|
// making the per-host breakdown a column of the owner's own name. Defaults to this node, so every
|
||||||
|
// caller that does not know about the mesh keeps its previous meaning exactly.
|
||||||
|
function vv_ai_token_record(string $profile, string $source, int $prompt, int $completion,
|
||||||
|
?float $tokS, string $node = ''): void {
|
||||||
if ($prompt <= 0 && $completion <= 0) return; // nothing generated; not a turn worth a row
|
if ($prompt <= 0 && $completion <= 0) return; // nothing generated; not a turn worth a row
|
||||||
|
|
||||||
$db = vv_ai_token_db();
|
$db = vv_ai_token_db();
|
||||||
$dir = dirname($db);
|
$dir = dirname($db);
|
||||||
if (!is_dir($dir) && !@mkdir($dir, 0755, true)) return;
|
if (!is_dir($dir) && !@mkdir($dir, 0755, true)) return;
|
||||||
|
|
||||||
|
// Validated, not trusted: the host column is what the per-host totals group on, and a value
|
||||||
|
// that is not a host slot would create a column nobody can place.
|
||||||
|
$node = preg_match('/^host\d+$/', $node) ? $node : vv_detect_host();
|
||||||
|
|
||||||
$row = implode('|', [
|
$row = implode('|', [
|
||||||
date('Y-m-d'), date('H:i:s'),
|
date('Y-m-d'), date('H:i:s'),
|
||||||
vv_detect_host(),
|
$node,
|
||||||
preg_replace('/[^a-z0-9_-]/i', '', $profile) ?: 'unknown',
|
preg_replace('/[^a-z0-9_-]/i', '', $profile) ?: 'unknown',
|
||||||
preg_replace('/[^a-z0-9_-]/i', '', $source) ?: 'unknown',
|
preg_replace('/[^a-z0-9_-]/i', '', $source) ?: 'unknown',
|
||||||
max(0, $prompt), max(0, $completion),
|
max(0, $prompt), max(0, $completion),
|
||||||
@@ -938,13 +1009,20 @@ function vv_ai_token_stats(): array {
|
|||||||
'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank];
|
'synced' => null, 'today' => $blank, 'week' => $blank, 'all' => $blank];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ledgers to read: our own, plus whatever AI/ai_token_sync.sh has pulled from partners.
|
// Ledgers to read: our own, plus any partner ledger still cached from when each node kept its
|
||||||
|
// own. The mesh shares one AI now — every turn generates on the owner and is recorded in the
|
||||||
|
// owner's ledger against the node that asked — so on a current install this loop finds
|
||||||
|
// nothing and the local file already holds the whole fleet.
|
||||||
|
//
|
||||||
|
// Kept rather than deleted: a cache from before the change still describes real turns, and
|
||||||
|
// dropping the read would silently retire history the tab is showing today.
|
||||||
//
|
//
|
||||||
// Each entry carries the host slot it is allowed to contribute rows for. A partner file may
|
// Each entry carries the host slot it is allowed to contribute rows for. A partner file may
|
||||||
// only add rows whose host column matches its filename — a ledger copied into the wrong slot,
|
// only add rows whose host column matches its filename — a ledger copied into the wrong slot,
|
||||||
// or a partner that somehow cached ours, would otherwise be counted twice against a total
|
// or a partner that somehow cached ours, would otherwise be counted twice against a total
|
||||||
// that still looked plausible. Our own file is trusted for any host, because it is the only
|
// that still looked plausible. Our own file is trusted for any host, because it is the only
|
||||||
// one written here and its host column is written by vv_detect_host().
|
// one written here and its host column is written by vv_ai_token_record(), which validates
|
||||||
|
// the slot before it writes one.
|
||||||
$ledgers = [[$db, null]];
|
$ledgers = [[$db, null]];
|
||||||
foreach ((array)@glob(VV_AI_TOKEN_CACHE_DIR . '/host*.tokens.db') as $partnerDb) {
|
foreach ((array)@glob(VV_AI_TOKEN_CACHE_DIR . '/host*.tokens.db') as $partnerDb) {
|
||||||
if (!preg_match('/(host\d+)\.tokens\.db$/', $partnerDb, $m)) continue;
|
if (!preg_match('/(host\d+)\.tokens\.db$/', $partnerDb, $m)) continue;
|
||||||
@@ -1405,6 +1483,114 @@ function vv_ai_bug_report(array $b): string {
|
|||||||
// Strictly read-only, and there is no counterpart that changes any of it. Knowing a container is
|
// Strictly read-only, and there is no counterpart that changes any of it. Knowing a container is
|
||||||
// down is what lets an explanation be about this machine instead of about Unraid in general;
|
// down is what lets an explanation be about this machine instead of about Unraid in general;
|
||||||
// restarting it is a decision that belongs to a person looking at the screen.
|
// restarting it is a decision that belongs to a person looking at the screen.
|
||||||
|
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// Fallback readiness, for the assistant on the Fallback tab.
|
||||||
|
//
|
||||||
|
// Fallback differs from every other subsystem here in one way that shapes this whole function: it
|
||||||
|
// is DORMANT until it isn't. A watchdog leaves strikes and restarts to reason about; fallback
|
||||||
|
// leaves nothing at all until a real outage, so "looks fine" and "would work" are unrelated. On
|
||||||
|
// 2026-08-23 the coverage card showed 12 containers configured and every one of them was absent
|
||||||
|
// from the partner — a failover would have started nothing, and no surface said so.
|
||||||
|
//
|
||||||
|
// So this reports what would ACTUALLY happen, not what is configured to happen, and it is explicit
|
||||||
|
// about the difference between the two.
|
||||||
|
//
|
||||||
|
// Never blocks on the network. Partner presence costs an SSH round trip per container, which is far
|
||||||
|
// too slow for a question already waiting on a model, so it is read from the cache
|
||||||
|
// coverage_deploy.sh --status writes and reported WITH ITS AGE. A stale answer stated as stale is
|
||||||
|
// useful; a stale answer stated as current is the failure this whole feature exists to prevent.
|
||||||
|
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
function vv_ai_fallback_state(): string {
|
||||||
|
$me = vv_detect_host();
|
||||||
|
if ($me === '') return '';
|
||||||
|
// vv_detect_host() returns the LOWERCASE slug (host1); the conf keys are uppercase
|
||||||
|
// (FALLBACK_HOST1_TIER1). Building the key from the slug as-is silently matched nothing and
|
||||||
|
// reported "NOTHING is covered" on a host with twelve covered containers — a confidently
|
||||||
|
// wrong answer, which is the one outcome this block must never produce.
|
||||||
|
$ME = strtoupper($me);
|
||||||
|
|
||||||
|
$conf = vv_read_conf_raw('master.conf');
|
||||||
|
if ($conf === '') return '';
|
||||||
|
|
||||||
|
$s = "FALLBACK READINESS (read-only — you cannot change any of it, and you must never tell the "
|
||||||
|
. "operator a failover will work unless the evidence below says so)\n";
|
||||||
|
|
||||||
|
// ── current state ───────────────────────────────────────────────────────────────────────
|
||||||
|
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||||
|
$state = 'unknown'; $since = '';
|
||||||
|
if (is_readable($stateFile)) {
|
||||||
|
$raw = (string) @file_get_contents($stateFile);
|
||||||
|
if (preg_match('/^state=(\S+)/m', $raw, $m)) $state = $m[1];
|
||||||
|
if (preg_match('/^fallback_start=(\d+)/m', $raw, $m) && (int)$m[1] > 0) {
|
||||||
|
$since = ' since ' . date('Y-m-d H:i', (int) $m[1]);
|
||||||
|
}
|
||||||
|
$age = time() - (int) @filemtime($stateFile);
|
||||||
|
// The steady NORMAL path writes nothing, so an old mtime is not staleness — it is quiet.
|
||||||
|
$s .= "- state: $state$since (state file last written "
|
||||||
|
. ($age < 3600 ? round($age / 60) . ' minutes' : round($age / 86400) . ' days') . " ago; "
|
||||||
|
. "the NORMAL path writes nothing, so an old file means nothing has changed)\n";
|
||||||
|
} else {
|
||||||
|
// No file is not the same as not running: fallback.sh writes only on a transition.
|
||||||
|
$live = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||||
|
$s .= $live
|
||||||
|
? "- state: NORMAL (inferred — the daemon is running and has never recorded a transition, "
|
||||||
|
. "so it has written no state file; this is health, not ignorance)\n"
|
||||||
|
: "- state: no state file AND no running daemon — fallback is not operating on this host\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (['FALLBACK_ENABLED', 'FALLBACK_RSYNC_ENABLED'] as $k) {
|
||||||
|
if (preg_match('/^\s*' . $k . '\s*=\s*"?(\w+)"?/m', $conf, $m)) {
|
||||||
|
$s .= "- $k: {$m[1]}"
|
||||||
|
. ($k === 'FALLBACK_RSYNC_ENABLED' && $m[1] !== 'true'
|
||||||
|
? " <- handback writeback is OFF: anything the partner writes while covering "
|
||||||
|
. "for this host never comes home\n" : "\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── coverage, tier by tier, with the real delays ─────────────────────────────────────────
|
||||||
|
$hostConf = vv_read_conf_raw($me . '.conf');
|
||||||
|
$covered = [];
|
||||||
|
for ($t = 1; $t <= 4; $t++) {
|
||||||
|
$names = vv_parse_conf_list($hostConf, "FALLBACK_{$ME}_TIER{$t}");
|
||||||
|
if (!$names) continue;
|
||||||
|
$delay = '';
|
||||||
|
if ($t > 1 && preg_match('/^\s*' . $ME . '_TIER' . $t . '_DELAY\s*=\s*"?(\d+)/m', $hostConf, $m)) {
|
||||||
|
$delay = " after {$m[1]} minutes";
|
||||||
|
}
|
||||||
|
$s .= "- tier $t" . ($t === 1 ? ' (immediate)' : $delay) . ': ' . implode(', ', $names) . "\n";
|
||||||
|
foreach ($names as $n) $covered[] = $n;
|
||||||
|
}
|
||||||
|
if (!$covered) {
|
||||||
|
$s .= "- coverage: NOTHING is covered — a failover would start no containers at all\n";
|
||||||
|
return $s . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── does the partner actually have them ─────────────────────────────────────────────────
|
||||||
|
$cache = '/tmp/varaverk/api/fallback_presence.json';
|
||||||
|
if (is_readable($cache)) {
|
||||||
|
$j = json_decode((string) @file_get_contents($cache), true);
|
||||||
|
$age = time() - (int) @filemtime($cache);
|
||||||
|
$miss = (array) ($j['missing'] ?? []);
|
||||||
|
$have = array_keys((array) ($j['present'] ?? []));
|
||||||
|
$when = $age < 3600 ? round($age / 60) . ' minutes ago' : round($age / 3600) . ' hours ago';
|
||||||
|
if ($miss) {
|
||||||
|
$s .= "- ON THE PARTNER (checked $when): " . count($miss) . ' of ' . count($covered)
|
||||||
|
. " covered container(s) DO NOT EXIST there: " . implode(', ', $miss) . "\n"
|
||||||
|
. " fallback.sh starts a covered container with `docker start`; it never creates one, "
|
||||||
|
. "so each of those would fail during a real outage. Push them from the Fallback "
|
||||||
|
. "coverage card.\n";
|
||||||
|
} else {
|
||||||
|
$s .= "- ON THE PARTNER (checked $when): all " . count($have)
|
||||||
|
. " covered container(s) exist there\n";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$s .= "- ON THE PARTNER: not checked. Say so plainly — whether a failover would actually "
|
||||||
|
. "start anything is UNKNOWN until the coverage card's presence check runs.\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
return $s . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
function vv_ai_system_state(): string {
|
function vv_ai_system_state(): string {
|
||||||
$p = '/tmp/varaverk/api/monitor.json';
|
$p = '/tmp/varaverk/api/monitor.json';
|
||||||
if (!is_readable($p)) return '';
|
if (!is_readable($p)) return '';
|
||||||
|
|||||||
@@ -0,0 +1,546 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Every AI action, in one dispatcher, independent of how the request arrived. api/ai.php calls
|
||||||
|
// it for browser requests; Tools/ai_rpc.php calls it for mesh requests forwarded from another
|
||||||
|
// node over SSH. Both get identical behaviour because there is only one implementation.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// Transport-free. Nothing here reads a superglobal, sets a header, echoes, or exits. Handlers
|
||||||
|
// take a params array and return the response body as an array. That is what makes the
|
||||||
|
// same code serveable over HTTP and over SSH — the handlers cannot tell the difference,
|
||||||
|
// so the two paths cannot drift.
|
||||||
|
//
|
||||||
|
// The dispatcher owns behaviour; the caller owns access.
|
||||||
|
// Authentication, CSRF and the node/action gates live in the caller. This file assumes
|
||||||
|
// the request is already allowed. Two callers with different trust models share one set
|
||||||
|
// of handlers precisely because the handlers do not re-litigate trust.
|
||||||
|
//
|
||||||
|
// Extracted, not rewritten.
|
||||||
|
// This was the body of api/ai.php. The transformation is mechanical — `echo json_encode(X);
|
||||||
|
// exit` became `return X`, a 405 became $httpStatus plus a return, and superglobals became
|
||||||
|
// $p. Every validation, every whitelist and every comment is the original. A refactor of a
|
||||||
|
// file this security-sensitive earns nothing by also being a redesign.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// Tokens are minted here and never accepted from elsewhere.
|
||||||
|
// random_bytes, hex, fixed length. A job's answer is readable by anyone who can guess its
|
||||||
|
// token, so the token is not guessable.
|
||||||
|
//
|
||||||
|
// Every token is validated as hex before it composes a path.
|
||||||
|
// vv_ai_job_path() returns null for anything else, and each caller checks. That pattern is
|
||||||
|
// what confines reads and deletes to the job directory.
|
||||||
|
//
|
||||||
|
// History is validated per message, not trusted as a blob.
|
||||||
|
// Role must be user or assistant, content must be a non-empty string, and each is
|
||||||
|
// truncated. A crafted history could otherwise inject a system role or push the context
|
||||||
|
// past the offload ceiling. chat_save applies the same validation, because a stored chat
|
||||||
|
// is replayed into a later prompt.
|
||||||
|
//
|
||||||
|
// Scope is whitelisted, not escaped.
|
||||||
|
// It reaches the model as text, so anything richer than a file name is an
|
||||||
|
// instruction-injection surface for no benefit.
|
||||||
|
//
|
||||||
|
// stop signals one verified pid, never a process group.
|
||||||
|
// The cmdline must name both the worker and this job's own token before anything is
|
||||||
|
// signalled, because pid reuse is real. Group signalling took the WebGUI down on
|
||||||
|
// 2026-08-07.
|
||||||
|
//
|
||||||
|
// EXPORTS
|
||||||
|
// vv_ai_dispatch() action → response body
|
||||||
|
// vv_ai_log() the shared request/action trace at /var/log/varaverk/ai.log
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/ai.php config, stats, memory, chats, bugs, job files
|
||||||
|
// include/ai_memory_learn.php loaded per action — learned-memory proposals
|
||||||
|
// include/ai_repair.php loaded per action — the largest include in the plugin
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
require_once __DIR__ . '/ai.php';
|
||||||
|
|
||||||
|
// History depth is per profile and still decided server-side rather than by the page — the page
|
||||||
|
// simply no longer carries a second copy of the numbers. include/ai_profiles.php holds them.
|
||||||
|
// Reasoning is not stored in history, so it does not compound.
|
||||||
|
const VV_AI_MAX_QUESTION = 4000; // characters
|
||||||
|
const VV_AI_MAX_HIST_MSG = 4000; // characters per retained message
|
||||||
|
const VV_AI_JOB_TTL = 3600; // seconds before a job file is reaped
|
||||||
|
|
||||||
|
// Request trace. There is no nginx access log on this host and the CSRF prepend exits with an
|
||||||
|
// empty body, so without this there is no way to tell "the request never arrived" from "the
|
||||||
|
// request arrived and failed" — which is exactly the ambiguity that made the first hang
|
||||||
|
// undiagnosable. Callers exclude poll, which would otherwise write a line per second per open tab.
|
||||||
|
function vv_ai_log(string $msg): void {
|
||||||
|
if (!is_dir('/var/log/varaverk')) return;
|
||||||
|
@file_put_contents('/var/log/varaverk/ai.log',
|
||||||
|
date('Y-m-d H:i:s') . ' ' . $msg . "\n", FILE_APPEND | LOCK_EX);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run one action. $p carries the request parameters regardless of how they arrived; $isPost is
|
||||||
|
// still passed separately because the POST-only checks are a CSRF guarantee, not a parameter.
|
||||||
|
//
|
||||||
|
// $httpStatus is by reference and named for what it is — `$status` is taken, by stop's own local
|
||||||
|
// for a job's state, and shadowing that would be a silent bug in the one handler that kills a
|
||||||
|
// process.
|
||||||
|
function vv_ai_dispatch(string $action, array $p, bool $isPost, int &$httpStatus = 200): array {
|
||||||
|
|
||||||
|
$postOnly = function () use (&$httpStatus): array {
|
||||||
|
$httpStatus = 405;
|
||||||
|
return ['ok' => false, 'error' => 'POST only'];
|
||||||
|
};
|
||||||
|
|
||||||
|
// ── stats ─────────────────────────────────────────────────────────────────────
|
||||||
|
// Served from the shared 'ai' cache that Tools/api_cache_writer.sh refreshes every minute, on
|
||||||
|
// the same terms as the monitor and arrs payloads. This action is polled every 30 seconds by
|
||||||
|
// every open tab and used to pay a full collection each time — around a second, most of it
|
||||||
|
// spent waiting on Ollama and nvidia-smi — for numbers that only change when the writer runs.
|
||||||
|
//
|
||||||
|
// live=1 bypasses it, for the case where something was just changed and the point is to see
|
||||||
|
// the result. A missing cache always falls back to collecting, so the cache can never be the
|
||||||
|
// reason the banner fails to render.
|
||||||
|
if ($action === 'stats') {
|
||||||
|
return ['ok' => true, 'stats' => vv_ai_stats_cached(isset($p['live']))];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── tokens ────────────────────────────────────────────────────────────────────
|
||||||
|
// Separate from stats rather than folded into it. stats is polled every 30 seconds by every
|
||||||
|
// open tab; this reads a file that grows without bound between prunes. The totals only move
|
||||||
|
// when a turn completes, and the page knows exactly when that happened, so it asks then.
|
||||||
|
if ($action === 'tokens') {
|
||||||
|
return ['ok' => true, 'tokens' => vv_ai_token_stats()];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── poll ──────────────────────────────────────────────────────────────────────
|
||||||
|
if ($action === 'poll') {
|
||||||
|
$token = trim($p['token'] ?? '');
|
||||||
|
if (vv_ai_job_path($token) === null) {
|
||||||
|
return ['ok' => false, 'error' => 'Invalid token'];
|
||||||
|
}
|
||||||
|
$job = vv_ai_job_read($token);
|
||||||
|
if ($job === null) {
|
||||||
|
// The worker writes its first state after this request may already have arrived.
|
||||||
|
return ['ok' => true, 'job' => ['status' => 'pending']];
|
||||||
|
}
|
||||||
|
return ['ok' => true, 'job' => $job];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── memory ────────────────────────────────────────────────────────────────────
|
||||||
|
if ($action === 'memory_get') {
|
||||||
|
$m = vv_ai_memory_read();
|
||||||
|
return ['ok' => true, 'memory' => $m['text'], 'chars' => $m['chars'],
|
||||||
|
'max' => vv_ai_memory_max(), 'exists' => $m['exists'],
|
||||||
|
'path' => vv_ai_memory_path()];
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'memory_set') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
$r = vv_ai_memory_write((string)($p['memory'] ?? ''));
|
||||||
|
vv_ai_log('memory_set ' . ($r['ok'] ? 'ok chars=' . $r['chars'] : 'FAILED: ' . $r['error']));
|
||||||
|
return $r + ['max' => vv_ai_memory_max()];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── learned-memory proposals ──────────────────────────────────────────────────
|
||||||
|
// The store the assistant files candidates into. Accepting is the only path by which
|
||||||
|
// model-written text reaches a prompt, and it is a POST so the CSRF prepend covers it.
|
||||||
|
if ($action === 'mem_proposals') {
|
||||||
|
require_once __DIR__ . '/ai_memory_learn.php';
|
||||||
|
$m = vv_ai_memory_read('learned');
|
||||||
|
return [
|
||||||
|
'ok' => true,
|
||||||
|
'enabled' => vv_ai_mem_learn_enabled(),
|
||||||
|
'auto' => vv_ai_mem_learn_auto(),
|
||||||
|
// The list states the gate as well as the rows: an empty list means "nothing proposed"
|
||||||
|
// when learning is on and "nothing is looking" when it is off, and those are different.
|
||||||
|
'open' => vv_ai_mem_list('open'),
|
||||||
|
'recent' => array_slice(vv_ai_mem_list(), 0, 25),
|
||||||
|
'learned' => ['chars' => $m['chars'], 'max' => vv_ai_memory_learned_max()],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'mem_proposal_action') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
require_once __DIR__ . '/ai_memory_learn.php';
|
||||||
|
$id = trim($p['id'] ?? '');
|
||||||
|
$act = trim($p['act'] ?? '');
|
||||||
|
$r = vv_ai_mem_action($id, $act);
|
||||||
|
vv_ai_log(sprintf('mem_proposal id=%s act=%s %s', $id, $act,
|
||||||
|
$r['ok'] ? 'ok' : ('FAILED: ' . ($r['error'] ?? '?'))));
|
||||||
|
return $r;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── stop ──────────────────────────────────────────────────────────────────────
|
||||||
|
// Cancels a generation in flight. Only ever signals ONE pid, verified to be the worker for
|
||||||
|
// this exact job — never a process group. Signalling a group is what took the WebGUI down on
|
||||||
|
// 2026-08-07, and no group kill is needed here: the worker is a single php process whose only
|
||||||
|
// child-like thing is an HTTP connection to Ollama, which dies with it.
|
||||||
|
//
|
||||||
|
// Whatever was already generated is kept. A turn stopped at 80% is usually stopped because the
|
||||||
|
// operator has seen enough, not because they want it discarded.
|
||||||
|
if ($action === 'stop') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
|
||||||
|
$token = trim($p['token'] ?? '');
|
||||||
|
if (vv_ai_job_path($token) === null) {
|
||||||
|
return ['ok' => false, 'error' => 'Invalid token'];
|
||||||
|
}
|
||||||
|
|
||||||
|
$job = vv_ai_job_read($token);
|
||||||
|
if ($job === null) return ['ok' => false, 'error' => 'No such job'];
|
||||||
|
|
||||||
|
$status = (string)($job['status'] ?? '');
|
||||||
|
if ($status === 'done' || $status === 'error' || $status === 'stopped') {
|
||||||
|
return ['ok' => true, 'already' => true, 'status' => $status];
|
||||||
|
}
|
||||||
|
|
||||||
|
$pid = (int)($job['pid'] ?? 0);
|
||||||
|
// Below 2 is init or nonsense. A pid we cannot verify is a pid we do not signal.
|
||||||
|
$killed = false;
|
||||||
|
if ($pid >= 2) {
|
||||||
|
// Pid reuse is the reason for this: the recorded worker may have exited seconds ago
|
||||||
|
// and the number been handed to something else entirely. The cmdline must name both
|
||||||
|
// this worker and this job's own file before anything is signalled.
|
||||||
|
$cmdline = @file_get_contents("/proc/$pid/cmdline");
|
||||||
|
$cmdline = $cmdline === false ? '' : str_replace("\0", ' ', $cmdline);
|
||||||
|
if (strpos($cmdline, 'ai_chat_worker.php') !== false && strpos($cmdline, $token) !== false) {
|
||||||
|
$killed = @posix_kill($pid, SIGTERM);
|
||||||
|
// No escalation ladder. The worker holds no lock and writes the job file
|
||||||
|
// atomically, so there is no cleanup that a delay would protect — and a SIGKILL
|
||||||
|
// race could land between the temp write and the rename.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The job file is rewritten either way. If the pid could not be verified the worker is
|
||||||
|
// already gone, and the page still needs a terminal state instead of polling to its
|
||||||
|
// ceiling.
|
||||||
|
$job['status'] = 'stopped';
|
||||||
|
$job['stopped'] = true;
|
||||||
|
$job['answer'] = trim((string)($job['partial'] ?? $job['answer'] ?? ''));
|
||||||
|
unset($job['partial']);
|
||||||
|
@file_put_contents(vv_ai_job_path($token), json_encode($job));
|
||||||
|
|
||||||
|
vv_ai_log(sprintf('stop token=%s pid=%d signalled=%s kept=%d chars',
|
||||||
|
substr($token, 0, 12), $pid, $killed ? 'yes' : 'no', strlen($job['answer'])));
|
||||||
|
|
||||||
|
return ['ok' => true, 'signalled' => $killed, 'kept' => strlen($job['answer'])];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── clear ─────────────────────────────────────────────────────────────────────
|
||||||
|
if ($action === 'clear') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
$path = vv_ai_job_path(trim($p['token'] ?? ''));
|
||||||
|
if ($path === null) return ['ok' => false, 'error' => 'Invalid token'];
|
||||||
|
if (file_exists($path)) @unlink($path);
|
||||||
|
return ['ok' => true];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── chats ─────────────────────────────────────────────────────────────────────
|
||||||
|
// Stored conversations. Listing and reading are GET because they change nothing; saving and
|
||||||
|
// deleting are POST, so they ride Unraid's CSRF prepend like every other mutation here.
|
||||||
|
//
|
||||||
|
// Messages are validated per message on the way in, exactly as ask validates history and for
|
||||||
|
// the same reason: a stored chat is replayed into a later prompt when the operator reopens it,
|
||||||
|
// so a crafted role in the store would be an injection that survives a reload.
|
||||||
|
if ($action === 'chats') {
|
||||||
|
return ['ok' => true, 'chats' => vv_ai_chats_list(), 'max' => vv_ai_chats_max()];
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'chat_get') {
|
||||||
|
$chat = vv_ai_chat_read(trim($p['id'] ?? ''));
|
||||||
|
if ($chat === null) return ['ok' => false, 'error' => 'No such chat'];
|
||||||
|
return ['ok' => true, 'chat' => $chat];
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'chat_save') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
|
||||||
|
$profile = trim($p['profile'] ?? 'chat');
|
||||||
|
if (!vv_ai_profile_ok($profile)) {
|
||||||
|
return ['ok' => false, 'error' => 'Unknown profile: ' . $profile];
|
||||||
|
}
|
||||||
|
|
||||||
|
$clean = [];
|
||||||
|
$msgs = json_decode($p['messages'] ?? '[]', true);
|
||||||
|
if (is_array($msgs)) {
|
||||||
|
foreach ($msgs as $m) {
|
||||||
|
$role = $m['role'] ?? '';
|
||||||
|
$text = trim((string)($m['content'] ?? ''));
|
||||||
|
if (!in_array($role, ['user', 'assistant'], true) || $text === '') continue;
|
||||||
|
$clean[] = ['role' => $role, 'content' => mb_substr($text, 0, VV_AI_MAX_HIST_MSG)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Capped at the deepest profile's window rather than that of the profile in hand. A chat
|
||||||
|
// saved under one profile can be reopened under another, and the reopened turn is trimmed
|
||||||
|
// again on the way back out by ask — so storing a little more than any single profile will
|
||||||
|
// send costs nothing and keeps the transcript readable.
|
||||||
|
$cap = vv_ai_profiles_max_turns() * 2;
|
||||||
|
if (count($clean) > $cap) $clean = array_slice($clean, -$cap);
|
||||||
|
|
||||||
|
// Whitelisted exactly as ask's is, and for the same reason: a scope is only ever a name
|
||||||
|
// from a page's own view state, it is stored and later replayed into a prompt, and
|
||||||
|
// anything richer than a file name is an instruction-injection surface for no benefit.
|
||||||
|
$scope = trim($p['scope'] ?? '');
|
||||||
|
if ($scope !== '' && !vv_ai_scope_ok($scope)) $scope = '';
|
||||||
|
|
||||||
|
$r = vv_ai_chat_save(trim($p['id'] ?? ''), $profile, $clean, $scope);
|
||||||
|
vv_ai_log('chat_save ' . ($r['ok'] ? 'ok id=' . substr($r['id'], 0, 12)
|
||||||
|
: 'FAILED: ' . $r['error']));
|
||||||
|
return $r;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($action === 'chat_delete') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
$ok = vv_ai_chat_delete(trim($p['id'] ?? ''));
|
||||||
|
return ['ok' => $ok, 'error' => $ok ? null : 'No such chat'];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── bugs / bug_close ──────────────────────────────────────────────────────────
|
||||||
|
// Reports the troubleshooter filed. Listing is a GET because it changes nothing; dismissing is
|
||||||
|
// a POST, like every other mutation in this plugin.
|
||||||
|
if ($action === 'bugs') {
|
||||||
|
return ['ok' => true, 'bugs' => vv_ai_bugs_list(($p['all'] ?? '') !== '1')];
|
||||||
|
}
|
||||||
|
if ($action === 'bug_close') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
$ok = vv_ai_bug_set_open(trim($p['id'] ?? ''), ($p['open'] ?? '0') === '1');
|
||||||
|
return ['ok' => $ok];
|
||||||
|
}
|
||||||
|
|
||||||
|
// The report, rendered server-side. Read-only by design: what the operator reviews is byte for
|
||||||
|
// byte what gets sent, so approving one text and transmitting another is not possible. It is
|
||||||
|
// also the only renderer — the page used to build its own markdown, which is two formats to
|
||||||
|
// keep in step and one of them always losing.
|
||||||
|
if ($action === 'bug_report') {
|
||||||
|
$id = trim($p['id'] ?? '');
|
||||||
|
$bug = null;
|
||||||
|
foreach (vv_ai_bugs_list(false) as $b) if (($b['id'] ?? '') === $id) { $bug = $b; break; }
|
||||||
|
if (!$bug) return ['ok' => false, 'error' => 'no such report'];
|
||||||
|
|
||||||
|
$t = vv_ai_bug_targets();
|
||||||
|
$title = '[' . ($bug['component'] ?? '?') . '] ' . ($bug['summary'] ?? '');
|
||||||
|
return [
|
||||||
|
'ok' => true,
|
||||||
|
'title' => $title,
|
||||||
|
'markdown' => vv_ai_bug_report($bug),
|
||||||
|
'targets' => $t,
|
||||||
|
// Built here because the repo name lives here. Length is the caller's problem to
|
||||||
|
// notice: GitHub truncates a very long query rather than refusing it, which would
|
||||||
|
// silently send a half report — so the page checks and falls back to the copy box.
|
||||||
|
'github' => 'https://github.com/' . $t['github_repo'] . '/issues/new?title='
|
||||||
|
. rawurlencode($title) . '&body=' . rawurlencode(vv_ai_bug_report($bug)),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sends to the operator's own Gitea, and only there. Never falls back to GitHub on failure:
|
||||||
|
// the two destinations are different people, and a silent substitution is how a report meant
|
||||||
|
// for a private backlog ends up public.
|
||||||
|
if ($action === 'bug_send_local') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
$id = trim($p['id'] ?? '');
|
||||||
|
$bug = null;
|
||||||
|
foreach (vv_ai_bugs_list(false) as $b) if (($b['id'] ?? '') === $id) { $bug = $b; break; }
|
||||||
|
if (!$bug) return ['ok' => false, 'error' => 'no such report'];
|
||||||
|
|
||||||
|
// Re-rendered from the store rather than taken from the request. The browser showed this
|
||||||
|
// text read-only; accepting a body from the page would make that guarantee decorative.
|
||||||
|
$r = vv_ai_bug_send_local('[' . ($bug['component'] ?? '?') . '] ' . ($bug['summary'] ?? ''),
|
||||||
|
vv_ai_bug_report($bug));
|
||||||
|
vv_ai_log(sprintf('bug_send_local id=%s %s', $id,
|
||||||
|
$r['ok'] ? 'ok ' . ($r['url'] ?? '') : 'failed: ' . ($r['error'] ?? '?')));
|
||||||
|
return $r;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── findings / finding_action ─────────────────────────────────────────────────
|
||||||
|
// What the repair sweep found, and the operator's answer to it. include/ai_repair.php is
|
||||||
|
// pulled in here rather than at the top of the file: it is the largest include in the plugin
|
||||||
|
// and poll runs once a second per open tab, so it is loaded by the two actions that need it
|
||||||
|
// and by nothing else.
|
||||||
|
//
|
||||||
|
// Neither action is gated on AI_REPAIR_ENABLED. Findings filed while it was on do not stop
|
||||||
|
// being true when it goes off, and answering them — including saying "this was never a
|
||||||
|
// problem" — is exactly what an operator turning the feature off is likely to want to do
|
||||||
|
// first. The gate states are reported instead, so the card can say what is running rather
|
||||||
|
// than the endpoint pretending the store is empty.
|
||||||
|
if ($action === 'findings' || $action === 'finding_action') {
|
||||||
|
require_once __DIR__ . '/ai_repair.php';
|
||||||
|
|
||||||
|
if ($action === 'findings') {
|
||||||
|
// Closed findings are the history — what was dismissed, what a fix actually fixed —
|
||||||
|
// and they are asked for explicitly rather than shipped with every poll of the list.
|
||||||
|
$rows = [];
|
||||||
|
$open = 0; $needs = 0;
|
||||||
|
foreach (vv_ai_findings_list(($p['all'] ?? '') === '1' ? [] : ['open', 'needs_operator']) as $f) {
|
||||||
|
$state = (string)($f['state'] ?? 'open');
|
||||||
|
if ($state === 'open') $open++;
|
||||||
|
elseif ($state === 'needs_operator') $needs++;
|
||||||
|
// The three things the page must not decide for itself: which actions this row
|
||||||
|
// offers, and what its state and kind mean in words.
|
||||||
|
$f['actions'] = vv_ai_finding_actions($f);
|
||||||
|
$f['state_label'] = VV_AI_FINDING_STATES[$state] ?? '';
|
||||||
|
$f['kind_label'] = VV_AI_FINDING_KINDS[(string)($f['kind'] ?? '')] ?? '';
|
||||||
|
$rows[] = $f;
|
||||||
|
}
|
||||||
|
return ['ok' => true,
|
||||||
|
'repair' => ['enabled' => vv_ai_repair_enabled(),
|
||||||
|
'autofix' => vv_ai_repair_autofix_enabled(),
|
||||||
|
'last' => vv_ai_sweep_last()],
|
||||||
|
'findings' => $rows,
|
||||||
|
'counts' => ['open' => $open, 'needs_operator' => $needs, 'shown' => count($rows)]];
|
||||||
|
}
|
||||||
|
|
||||||
|
// POST, because fix writes conf through the guarded path and every other answer writes
|
||||||
|
// state. Which actions are legal for a given row is vv_ai_finding_apply_action()'s call,
|
||||||
|
// not this endpoint's — a tab left open overnight is holding buttons the store has moved
|
||||||
|
// past.
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
|
||||||
|
$fid = trim($p['id'] ?? '');
|
||||||
|
$act = trim($p['act'] ?? '');
|
||||||
|
$r = vv_ai_finding_apply_action($fid, $act, trim($p['note'] ?? ''));
|
||||||
|
vv_ai_log(sprintf('finding_action id=%s act=%s %s', $fid, $act,
|
||||||
|
$r['ok'] ? 'ok' : 'FAILED: ' . ($r['error'] ?? '?')));
|
||||||
|
return $r;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── finding_write ─────────────────────────────────────────────────────────────
|
||||||
|
// A sweep on another node filing what it found. Not reachable from a browser — vv_ai_route()
|
||||||
|
// never returns LOCAL for it off the owner and the page has no caller — it exists so that
|
||||||
|
// "sweep local, store central" needs no second store and no reconciliation.
|
||||||
|
//
|
||||||
|
// The host is taken from the transport's own view of who connected, never from the payload.
|
||||||
|
// The record decides which machine a fault is about and is what the finding id hashes on, so
|
||||||
|
// letting the body name it would let one node file findings as another.
|
||||||
|
if ($action === 'finding_write') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
require_once __DIR__ . '/ai_repair.php';
|
||||||
|
|
||||||
|
$f = json_decode((string)($p['finding'] ?? ''), true);
|
||||||
|
if (!is_array($f)) return ['ok' => false, 'error' => 'finding_write: unreadable finding'];
|
||||||
|
|
||||||
|
$node = trim((string)($p['_vv_node'] ?? ''));
|
||||||
|
if (!preg_match('/^host\d+$/', $node)) {
|
||||||
|
return ['ok' => false, 'error' => 'finding_write: caller did not identify a node'];
|
||||||
|
}
|
||||||
|
$f['host'] = $node;
|
||||||
|
|
||||||
|
$r = vv_ai_finding_write_local($f);
|
||||||
|
vv_ai_log(sprintf('finding_write from=%s kind=%s %s', $node, (string)($f['kind'] ?? '?'),
|
||||||
|
($r['ok'] ?? false) ? 'ok id=' . ($r['id'] ?? '?') : 'FAILED: ' . ($r['error'] ?? '?')));
|
||||||
|
return $r;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── incident_add ──────────────────────────────────────────────────────────────
|
||||||
|
// Appends one operator-written "this was the fix" note against a scope. POST only, and the
|
||||||
|
// scope is whitelisted the same way ask's is — it is written to a file that later rides in a
|
||||||
|
// prompt, so it gets the same treatment as anything else that reaches the model.
|
||||||
|
if ($action === 'incident_add') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
return vv_ai_incident_add(
|
||||||
|
trim($p['scope'] ?? ''), trim($p['symptom'] ?? ''), trim($p['fix'] ?? ''));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── ask ───────────────────────────────────────────────────────────────────────
|
||||||
|
if ($action === 'ask') {
|
||||||
|
if (!$isPost) return $postOnly();
|
||||||
|
|
||||||
|
$cfg = vv_ai_config();
|
||||||
|
if ($cfg['model'] === '') {
|
||||||
|
return ['ok' => false, 'error' => 'No generation model configured'];
|
||||||
|
}
|
||||||
|
|
||||||
|
$question = trim($p['question'] ?? '');
|
||||||
|
if ($question === '') return ['ok' => false, 'error' => 'question is required'];
|
||||||
|
if (mb_strlen($question) > VV_AI_MAX_QUESTION) {
|
||||||
|
return ['ok' => false, 'error' => 'question exceeds ' . VV_AI_MAX_QUESTION . ' characters'];
|
||||||
|
}
|
||||||
|
|
||||||
|
$profile = trim($p['profile'] ?? 'varaverk');
|
||||||
|
if (!vv_ai_profile_ok($profile)) {
|
||||||
|
return ['ok' => false, 'error' => 'Unknown profile: ' . $profile];
|
||||||
|
}
|
||||||
|
$maxTurns = vv_ai_profile_turns($profile);
|
||||||
|
|
||||||
|
// Where the caller is standing — "master.conf", "daily_sync_maintenance.sh", a log name.
|
||||||
|
// The scheduler page sends it so a question can say "this setting" and mean something;
|
||||||
|
// the AI tab sends nothing and the worker simply omits the location line.
|
||||||
|
//
|
||||||
|
// Whitelisted hard, not escaped and hoped for. It reaches the model as text, so anything
|
||||||
|
// richer than a file name is an instruction-injection surface for no benefit — a scope is
|
||||||
|
// only ever a name from this page's own view state.
|
||||||
|
$scope = trim($p['scope'] ?? '');
|
||||||
|
if ($scope !== '' && !vv_ai_scope_ok($scope)) $scope = '';
|
||||||
|
|
||||||
|
// The retrieval filter only means anything to the profile that retrieves.
|
||||||
|
$kind = vv_ai_profile_can($profile, 'kind_filter') ? trim($p['kind'] ?? '') : '';
|
||||||
|
if ($kind !== '' && !in_array($kind, VV_AI_KINDS, true)) {
|
||||||
|
return ['ok' => false, 'error' => 'Unknown kind: ' . $kind];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate per message rather than trusting the blob: a crafted history could otherwise
|
||||||
|
// inject a system role, or push the context past the offload ceiling.
|
||||||
|
$clean = [];
|
||||||
|
$hist = json_decode($p['history'] ?? '[]', true);
|
||||||
|
if (is_array($hist)) {
|
||||||
|
foreach ($hist as $m) {
|
||||||
|
$role = $m['role'] ?? '';
|
||||||
|
$text = trim((string)($m['content'] ?? ''));
|
||||||
|
if (!in_array($role, ['user', 'assistant'], true) || $text === '') continue;
|
||||||
|
$clean[] = ['role' => $role, 'content' => mb_substr($text, 0, VV_AI_MAX_HIST_MSG)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (count($clean) > $maxTurns * 2) {
|
||||||
|
$clean = array_slice($clean, -($maxTurns * 2));
|
||||||
|
}
|
||||||
|
|
||||||
|
$dir = vv_ai_job_dir();
|
||||||
|
foreach (glob($dir . '/*.json') ?: [] as $old) {
|
||||||
|
if (time() - (int)@filemtime($old) > VV_AI_JOB_TTL) @unlink($old);
|
||||||
|
}
|
||||||
|
|
||||||
|
$token = bin2hex(random_bytes(16));
|
||||||
|
$jobFile = vv_ai_job_path($token);
|
||||||
|
$worker = dirname(__DIR__) . '/Tools/ai_chat_worker.php';
|
||||||
|
|
||||||
|
if (!file_exists($worker)) {
|
||||||
|
return ['ok' => false, 'error' => 'ai_chat_worker.php not found'];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not suppressed: if the job file cannot be written the worker has nowhere to report and
|
||||||
|
// the page polls a token that will never resolve — which looks exactly like a hang.
|
||||||
|
if (file_put_contents($jobFile, json_encode(['status' => 'pending'])) === false) {
|
||||||
|
vv_ai_log('ask FAILED — cannot write ' . $jobFile);
|
||||||
|
return ['ok' => false, 'error' => 'Cannot write job file to ' . VV_AI_JOB_DIR];
|
||||||
|
}
|
||||||
|
|
||||||
|
// setsid, not just nohup. nohup detaches from the terminal but leaves the child in the
|
||||||
|
// caller's process group — php-fpm's. That is the arrangement that took the WebGUI down
|
||||||
|
// on 2026-08-07 when a Stop signalled a group it did not own. Stop above signals one
|
||||||
|
// verified pid and never a group, so this is belt and braces, but it also means a php-fpm
|
||||||
|
// restart no longer takes a running generation with it.
|
||||||
|
$cmd = 'setsid nohup php ' . escapeshellarg($worker) . ' '
|
||||||
|
. escapeshellarg($jobFile) . ' '
|
||||||
|
. escapeshellarg($question) . ' '
|
||||||
|
. escapeshellarg(json_encode($clean)) . ' '
|
||||||
|
. escapeshellarg($kind) . ' '
|
||||||
|
. escapeshellarg(($p['think'] ?? '1') === '1' ? '1' : '0') . ' '
|
||||||
|
. escapeshellarg($profile) . ' '
|
||||||
|
. escapeshellarg($scope) . ' '
|
||||||
|
// Asked for per turn. Only meaningful on a profile holding web_search — the worker
|
||||||
|
// checks that, so a crafted web=1 against any other profile changes nothing.
|
||||||
|
. escapeshellarg(($p['web'] ?? '') === '1' ? '1' : '0') . ' '
|
||||||
|
// Who asked. Set by the RPC layer for a forwarded turn and absent for a local one,
|
||||||
|
// where the worker's own default is already correct. This is what keeps the token
|
||||||
|
// ledger's per-host column meaningful once every turn generates on the owner.
|
||||||
|
. escapeshellarg((string)($p['_vv_node'] ?? ''))
|
||||||
|
. ' >/dev/null 2>&1 </dev/null &';
|
||||||
|
$out = []; $rc = 0;
|
||||||
|
exec($cmd, $out, $rc);
|
||||||
|
// Redacted before it is logged, for the same reason the stored transcript is: asking the
|
||||||
|
// assistant to set a credential means typing one, and ai.log is neither 0600 nor pruned.
|
||||||
|
vv_ai_log(sprintf('ask token=%s rc=%d profile=%s kind=%s q=%s',
|
||||||
|
substr($token, 0, 12), $rc, $profile, $kind ?: '-',
|
||||||
|
mb_substr(vv_ai_redact($question), 0, 80)));
|
||||||
|
|
||||||
|
return ['ok' => true, 'token' => $token];
|
||||||
|
}
|
||||||
|
|
||||||
|
return ['ok' => false, 'error' => 'Unknown action'];
|
||||||
|
}
|
||||||
@@ -74,6 +74,9 @@
|
|||||||
// so relying on something else having loaded the registry first works on the AI tab and fatals
|
// so relying on something else having loaded the registry first works on the AI tab and fatals
|
||||||
// on the dashboard.
|
// on the dashboard.
|
||||||
require_once __DIR__ . '/ai_profiles.php';
|
require_once __DIR__ . '/ai_profiles.php';
|
||||||
|
// The mesh pane the chat card can switch to. Required here rather than by each page, so a page
|
||||||
|
// that mounts a chat gets the option without having to know the store exists.
|
||||||
|
require_once __DIR__ . '/node_chat.php';
|
||||||
|
|
||||||
// The conversation store, on its own, with no styling and no chat widget attached.
|
// The conversation store, on its own, with no styling and no chat widget attached.
|
||||||
//
|
//
|
||||||
@@ -146,8 +149,28 @@ function vv_ai_chat_assets(): void {
|
|||||||
border-top:1px dashed #1e1e1e; padding-top:8px; }
|
border-top:1px dashed #1e1e1e; padding-top:8px; }
|
||||||
|
|
||||||
/* ── Transcript ─────────────────────────────────────────────────────────── */
|
/* ── Transcript ─────────────────────────────────────────────────────────── */
|
||||||
.vv-ai-chat { border:1px solid #262626; border-radius:6px; background:#0b0b0b;
|
/* Transparent so the mark on the wrapper shows through. The dark ground moves to the wrapper —
|
||||||
|
painting it here would put an opaque layer between the badge and the text. */
|
||||||
|
.vv-ai-chat { border:1px solid #262626; border-radius:6px; background:transparent;
|
||||||
|
position:relative; z-index:1;
|
||||||
min-height:340px; max-height:60vh; overflow-y:auto; padding:14px; }
|
min-height:340px; max-height:60vh; overflow-y:auto; padding:14px; }
|
||||||
|
|
||||||
|
/* The Varaverk mark, behind every conversation on every page.
|
||||||
|
On the wrapper rather than on the scroller: a background on .vv-ai-chat would be anchored to
|
||||||
|
its padding box and drift as the transcript scrolls, and a child element would scroll away with
|
||||||
|
the messages. inset:0 on the positioned wrapper holds it still behind whatever moves. */
|
||||||
|
.vv-ai-chat-wrap { background:#0b0b0b; border-radius:6px; }
|
||||||
|
/* Positioned box with a transform, not a background-position percentage.
|
||||||
|
The mark is wider than this transcript is tall, so as a background it overflows vertically —
|
||||||
|
and percentage positioning on an overflowing image runs BACKWARDS: raising the number moves it
|
||||||
|
up. Nudging it "down" from 74% to 86% visibly raised it. A sized box centred on a point behaves
|
||||||
|
the way the number reads, and matches how the mesh pane places the same image. */
|
||||||
|
.vv-ai-chat-wrap::before {
|
||||||
|
content:''; position:absolute; left:50%; top:66%; transform:translate(-50%,-50%);
|
||||||
|
width:min(84%,520px); aspect-ratio:1; z-index:0; pointer-events:none;
|
||||||
|
background:url('/plugins/varaverk/icons/varaverk.png') center/contain no-repeat;
|
||||||
|
opacity:.032; filter:grayscale(1);
|
||||||
|
}
|
||||||
.vv-ai-empty { color:#3a3a3a; font-size:12px; text-align:center; padding:60px 20px; line-height:1.7; }
|
.vv-ai-empty { color:#3a3a3a; font-size:12px; text-align:center; padding:60px 20px; line-height:1.7; }
|
||||||
.vv-ai-msg { margin-bottom:16px; }
|
.vv-ai-msg { margin-bottom:16px; }
|
||||||
.vv-ai-role { font-size:9px; letter-spacing:.08em; text-transform:uppercase; margin-bottom:5px; }
|
.vv-ai-role { font-size:9px; letter-spacing:.08em; text-transform:uppercase; margin-bottom:5px; }
|
||||||
@@ -186,8 +209,6 @@ function vv_ai_chat_assets(): void {
|
|||||||
|
|
||||||
/* Folded answers cap at a readable height with a hard bottom edge rather than a fade — a fade
|
/* Folded answers cap at a readable height with a hard bottom edge rather than a fade — a fade
|
||||||
over a code block reads as a rendering fault. */
|
over a code block reads as a rendering fault. */
|
||||||
.vv-ai-fold { max-height:420px; overflow:hidden; }
|
|
||||||
.vv-ai-more { display:block; margin:4px 0 2px; }
|
|
||||||
|
|
||||||
.vv-ai-last { display:flex; gap:10px; padding:0 2px 4px; }
|
.vv-ai-last { display:flex; gap:10px; padding:0 2px 4px; }
|
||||||
.vv-ai-lnk { background:none; border:none; padding:0; font-size:10px; color:#4a4a4a;
|
.vv-ai-lnk { background:none; border:none; padding:0; font-size:10px; color:#4a4a4a;
|
||||||
@@ -772,12 +793,32 @@ vv_ai_profiles_script();
|
|||||||
+ `<pre><code>${body}</code></pre></div>`;
|
+ `<pre><code>${body}</code></pre></div>`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The same palette the mesh composer offers, by NAME only. The model never supplies a colour
|
||||||
|
// value — it picks from this list and the hex is looked up here, so nothing it writes can
|
||||||
|
// become a style property. Same closed-set principle as vv_nc_clean_style() on the server.
|
||||||
|
const FMT_COLORS = { green:'#4caf50', blue:'#4a9eff', amber:'#ff9800',
|
||||||
|
red:'#ef5350', violet:'#ab7df6', grey:'#bdbdbd' };
|
||||||
|
|
||||||
// ── Minimal markdown, applied strictly after escaping ────────────────
|
// ── Minimal markdown, applied strictly after escaping ────────────────
|
||||||
function fmt(text) {
|
function fmt(text) {
|
||||||
let h = esc(text);
|
let h = esc(text);
|
||||||
h = h.replace(/```(\w*)\n([\s\S]*?)```/g, (m, l, c) => codeCard(l, c));
|
h = h.replace(/```(\w*)\n([\s\S]*?)```/g, (m, l, c) => codeCard(l, c));
|
||||||
h = h.replace(/`([^`\n]+)`/g, '<code>$1</code>');
|
h = h.replace(/`([^`\n]+)`/g, '<code>$1</code>');
|
||||||
h = h.replace(/\*\*([^*\n]+)\*\*/g, '<strong>$1</strong>');
|
h = h.replace(/\*\*([^*\n]+)\*\*/g, '<strong>$1</strong>');
|
||||||
|
|
||||||
|
// Emphasis and size. Single-asterisk italics run after bold so ** is already consumed.
|
||||||
|
h = h.replace(/(^|[^*])\*([^*\n]+)\*(?!\*)/g, '$1<em>$2</em>');
|
||||||
|
h = h.replace(/__([^_\n]+)__/g, '<u>$1</u>');
|
||||||
|
h = h.replace(/\{big\}([\s\S]*?)\{\/big\}/g, '<span style="font-size:15px">$1</span>');
|
||||||
|
h = h.replace(/\{small\}([\s\S]*?)\{\/small\}/g, '<span style="font-size:11px">$1</span>');
|
||||||
|
|
||||||
|
// {green}…{/green}. An unknown name is left as written rather than guessed at — a colour
|
||||||
|
// nobody chose is worse than a pair of braces the reader can see and ignore.
|
||||||
|
h = h.replace(/\{(\w+)\}([\s\S]*?)\{\/\1\}/g, (m, name, inner) => {
|
||||||
|
const c = FMT_COLORS[name.toLowerCase()];
|
||||||
|
return c ? `<span style="color:${c}">${inner}</span>` : m;
|
||||||
|
});
|
||||||
|
|
||||||
h = h.replace(/\[(\d+)\]/g, '<span class="vv-ai-cite" data-cite="$1">[$1]</span>');
|
h = h.replace(/\[(\d+)\]/g, '<span class="vv-ai-cite" data-cite="$1">[$1]</span>');
|
||||||
return h;
|
return h;
|
||||||
}
|
}
|
||||||
@@ -899,6 +940,17 @@ vv_ai_profiles_script();
|
|||||||
}
|
}
|
||||||
|
|
||||||
const SEE_THINK_KEY = 'vvAiSeeThink:' + P;
|
const SEE_THINK_KEY = 'vvAiSeeThink:' + P;
|
||||||
|
// Per instance, like the reasoning key beside it: the Monitor card and a tab's assistant are
|
||||||
|
// opened for different reasons and should not share one answer to "pick up where I left off".
|
||||||
|
const RESUME_KEY = 'vvAiResume:' + P;
|
||||||
|
// WHICH thread this card was last on, per card. The chat store is shared on purpose — a
|
||||||
|
// conversation started on the dashboard is the one you carry on in a tab — but "resume" was
|
||||||
|
// resolved as "the newest thread in this profile", and Fallback, Partnership and Monitor all
|
||||||
|
// pin the same profile. Opening any of them dragged all three onto whichever had been typed
|
||||||
|
// in last, so two cards could never hold two different conversations. The pool below is still
|
||||||
|
// the shared list; this only decides which of it this card comes back to.
|
||||||
|
const LAST_CHAT_KEY = 'vvAiLastChat:' + P;
|
||||||
|
const rememberChat = id => { try { id ? localStorage.setItem(LAST_CHAT_KEY, id) : localStorage.removeItem(LAST_CHAT_KEY); } catch (_) {} };
|
||||||
const seeThink = () => { const c = $('see-think'); return !!c && c.checked; };
|
const seeThink = () => { const c = $('see-think'); return !!c && c.checked; };
|
||||||
|
|
||||||
// Paths ride in data attributes rather than an onclick. They come out of the index, and a
|
// Paths ride in data attributes rather than an onclick. They come out of the index, and a
|
||||||
@@ -956,9 +1008,6 @@ vv_ai_profiles_script();
|
|||||||
}
|
}
|
||||||
const node = el(h + '</div>');
|
const node = el(h + '</div>');
|
||||||
chatEl().appendChild(node);
|
chatEl().appendChild(node);
|
||||||
// Measured after it is in the DOM — scrollHeight is 0 on a detached node, so a fold decided
|
|
||||||
// before appending would either never fire or fire on everything.
|
|
||||||
foldIfLong(node);
|
|
||||||
// Obeys Follow like every other write. This was the one place that did not, and it was the
|
// Obeys Follow like every other write. This was the one place that did not, and it was the
|
||||||
// worst place for it: unticking the box to hold your place while an answer arrives, only to
|
// worst place for it: unticking the box to hold your place while an answer arrives, only to
|
||||||
// be dragged to the bottom the instant it lands.
|
// be dragged to the bottom the instant it lands.
|
||||||
@@ -966,19 +1015,11 @@ vv_ai_profiles_script();
|
|||||||
syncJump();
|
syncJump();
|
||||||
}
|
}
|
||||||
|
|
||||||
// A long answer buries the composer on a 15" panel, and the composer is where the next thing
|
// Answers render in full. They used to fold at 420px behind a "Show the rest" control, on the
|
||||||
// happens. Folded to a readable height with the control always visible — never hover-revealed,
|
// reasoning that a long answer buries the composer on a 15" panel — true, but the cost was
|
||||||
// because the surfaces this runs on have no pointer near them.
|
// paid on every answer to save a scroll, and the fold hid the part of a long answer that is
|
||||||
const FOLD_PX = 420;
|
// usually the point. Scrolling is cheaper than a click plus a scroll, and Follow already
|
||||||
function foldIfLong(node) {
|
// handles keeping your place while an answer arrives.
|
||||||
const body = node.querySelector('.vv-ai-body');
|
|
||||||
if (!body || body.scrollHeight <= FOLD_PX) return;
|
|
||||||
body.classList.add('vv-ai-fold');
|
|
||||||
body.insertAdjacentHTML('afterend',
|
|
||||||
`<button type="button" class="vv-ai-lnk vv-ai-more" data-more>Show the rest`
|
|
||||||
+ ` (${Math.round(body.scrollHeight / 20)} lines)</button>`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function addError(msg) {
|
function addError(msg) {
|
||||||
const p = $('pending'); if (p) p.remove();
|
const p = $('pending'); if (p) p.remove();
|
||||||
chatEl().appendChild(el(`<div class="vv-ai-msg bot"><div class="vv-ai-role">Varaverk</div>`
|
chatEl().appendChild(el(`<div class="vv-ai-msg bot"><div class="vv-ai-role">Varaverk</div>`
|
||||||
@@ -1030,14 +1071,6 @@ vv_ai_profiles_script();
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const more = e.target.closest('[data-more]');
|
|
||||||
if (more) {
|
|
||||||
const body = more.previousElementSibling;
|
|
||||||
if (body) body.classList.remove('vv-ai-fold');
|
|
||||||
more.remove();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const stog = e.target.closest('[data-src-toggle]');
|
const stog = e.target.closest('[data-src-toggle]');
|
||||||
if (stog) {
|
if (stog) {
|
||||||
const box = stog.parentElement;
|
const box = stog.parentElement;
|
||||||
@@ -1330,6 +1363,15 @@ vv_ai_profiles_script();
|
|||||||
profile: profile,
|
profile: profile,
|
||||||
question: q,
|
question: q,
|
||||||
history: JSON.stringify(sendable()),
|
history: JSON.stringify(sendable()),
|
||||||
|
// Where the caller is standing, so a question can say "this setting" and mean it. The
|
||||||
|
// backend whitelists it and passes it to the worker, which turns it into a location
|
||||||
|
// line; a page that sets no scope sends '' and the worker omits that line.
|
||||||
|
//
|
||||||
|
// Keep this here. The scope-aware dock in 185abdb sent it from the page's own ask call;
|
||||||
|
// 8eeb4c3 folded that dock into this component and the line did not come with it, so
|
||||||
|
// for the whole of that window Scheduler computed a scope, re-read it at send time, and
|
||||||
|
// the model never saw it. Nothing errored — the answer just arrived ungrounded.
|
||||||
|
scope: (typeof o.scope === 'function' ? o.scope() : (o.scope || '')),
|
||||||
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
|
kind: (PROFILES[profile].kind && kindEl) ? kindEl.value : '',
|
||||||
// A checkbox where the page offers one, otherwise whatever the page decides from the
|
// A checkbox where the page offers one, otherwise whatever the page decides from the
|
||||||
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
|
// profile, otherwise on. The Scheduler reasons only when diagnosing: working out what a
|
||||||
@@ -1415,7 +1457,7 @@ vv_ai_profiles_script();
|
|||||||
fetch(API, { method: 'POST', headers: POST_HEAD,
|
fetch(API, { method: 'POST', headers: POST_HEAD,
|
||||||
body: new URLSearchParams({ action: 'stop', token: t }) })
|
body: new URLSearchParams({ action: 'stop', token: t }) })
|
||||||
.then(r => r.json())
|
.then(r => r.json())
|
||||||
.catch(() => {});
|
.catch(e => vvFetchErr('stop generation', e));
|
||||||
// Nothing is rendered from the response. The poll already owns turning a terminal state into
|
// Nothing is rendered from the response. The poll already owns turning a terminal state into
|
||||||
// a message, and having two paths do it is how a turn ends up in the transcript twice.
|
// a message, and having two paths do it is how a turn ends up in the transcript twice.
|
||||||
}
|
}
|
||||||
@@ -1440,7 +1482,7 @@ vv_ai_profiles_script();
|
|||||||
noteLine('Stopped before anything was written.');
|
noteLine('Stopped before anything was written.');
|
||||||
}
|
}
|
||||||
fetch(API, { method: 'POST', headers: POST_HEAD,
|
fetch(API, { method: 'POST', headers: POST_HEAD,
|
||||||
body: new URLSearchParams({ action: 'clear', token }) }).catch(() => {});
|
body: new URLSearchParams({ action: 'clear', token }) }).catch(e => vvFetchErr('clear job', e));
|
||||||
finish();
|
finish();
|
||||||
save();
|
save();
|
||||||
return;
|
return;
|
||||||
@@ -1450,7 +1492,7 @@ vv_ai_profiles_script();
|
|||||||
addAnswer(j);
|
addAnswer(j);
|
||||||
messages.push({ role: 'assistant', content: j.answer });
|
messages.push({ role: 'assistant', content: j.answer });
|
||||||
fetch(API, { method: 'POST', headers: POST_HEAD,
|
fetch(API, { method: 'POST', headers: POST_HEAD,
|
||||||
body: new URLSearchParams({ action: 'clear', token }) }).catch(() => {});
|
body: new URLSearchParams({ action: 'clear', token }) }).catch(e => vvFetchErr('clear job', e));
|
||||||
finish();
|
finish();
|
||||||
save();
|
save();
|
||||||
// The answer is passed so a page can react to what was said, not just that a turn
|
// The answer is passed so a page can react to what was said, not just that a turn
|
||||||
@@ -1493,8 +1535,8 @@ vv_ai_profiles_script();
|
|||||||
messages: JSON.stringify(messages),
|
messages: JSON.stringify(messages),
|
||||||
}) })
|
}) })
|
||||||
.then(r => r.json())
|
.then(r => r.json())
|
||||||
.then(d => { if (d.ok) { chatId = d.id; onChats(chatId); } })
|
.then(d => { if (d.ok) { chatId = d.id; rememberChat(chatId); onChats(chatId); } })
|
||||||
.catch(() => {});
|
.catch(e => vvFetchErr('save conversation', e));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1512,10 +1554,6 @@ vv_ai_profiles_script();
|
|||||||
+ `<div class="vv-ai-body">${fmt(m.content)}</div>`
|
+ `<div class="vv-ai-body">${fmt(m.content)}</div>`
|
||||||
+ (m.offer ? offerHtml(m.offer, i) : '') + `</div>`);
|
+ (m.offer ? offerHtml(m.offer, i) : '') + `</div>`);
|
||||||
c.appendChild(node);
|
c.appendChild(node);
|
||||||
// Folded here too, not only on arrival. A reopened thread carrying several long answers
|
|
||||||
// is exactly the case the fold exists for — it is what buries the composer on a 15"
|
|
||||||
// panel. Measured after the append, because scrollHeight is 0 on a detached node.
|
|
||||||
foldIfLong(node);
|
|
||||||
});
|
});
|
||||||
scroll();
|
scroll();
|
||||||
syncLast();
|
syncLast();
|
||||||
@@ -1577,6 +1615,7 @@ vv_ai_profiles_script();
|
|||||||
// end would send the model an empty history for a transcript full of context.
|
// end would send the model an empty history for a transcript full of context.
|
||||||
sendFrom = 0;
|
sendFrom = 0;
|
||||||
chatId = c.id || '';
|
chatId = c.id || '';
|
||||||
|
rememberChat(chatId);
|
||||||
lastSources = [];
|
lastSources = [];
|
||||||
if (PROFILES[c.profile]) applyProfile(c.profile);
|
if (PROFILES[c.profile]) applyProfile(c.profile);
|
||||||
render();
|
render();
|
||||||
@@ -1586,6 +1625,9 @@ vv_ai_profiles_script();
|
|||||||
|
|
||||||
function newChat() {
|
function newChat() {
|
||||||
messages = []; sendFrom = 0; chatId = ''; lastSources = [];
|
messages = []; sendFrom = 0; chatId = ''; lastSources = [];
|
||||||
|
// Forget, so New means new on the next visit too. Without this the card would reopen the
|
||||||
|
// thread the operator had just deliberately stepped away from.
|
||||||
|
rememberChat('');
|
||||||
reset();
|
reset();
|
||||||
onChats('');
|
onChats('');
|
||||||
}
|
}
|
||||||
@@ -1988,7 +2030,19 @@ vv_ai_profiles_script();
|
|||||||
wrapEl.addEventListener('keydown', e => {
|
wrapEl.addEventListener('keydown', e => {
|
||||||
const inp = $('input');
|
const inp = $('input');
|
||||||
|
|
||||||
if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { e.preventDefault(); send(); return; }
|
// Enter asks; Ctrl/Cmd+Enter breaks the line. The reverse of the usual editor convention,
|
||||||
|
// and right for this box: almost every message here is one line, so the common action
|
||||||
|
// gets the bare key and the rare one gets the modifier. Shift+Enter also breaks a line,
|
||||||
|
// because that is the reflex people arrive with from every other chat box.
|
||||||
|
if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) {
|
||||||
|
e.preventDefault();
|
||||||
|
const t = e.target;
|
||||||
|
const s = t.selectionStart ?? t.value.length, n = t.selectionEnd ?? s;
|
||||||
|
t.value = t.value.slice(0, s) + '\n' + t.value.slice(n);
|
||||||
|
t.selectionStart = t.selectionEnd = s + 1;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); send(); return; }
|
||||||
|
|
||||||
// Terminal habit: an empty box and Up gets back what you just asked, to edit rather than
|
// Terminal habit: an empty box and Up gets back what you just asked, to edit rather than
|
||||||
// retype. Only when empty, or it would eat cursor movement in a question being written.
|
// retype. Only when empty, or it would eat cursor movement in a question being written.
|
||||||
@@ -2056,7 +2110,20 @@ vv_ai_profiles_script();
|
|||||||
//
|
//
|
||||||
// Silent on failure. Nothing here is worth an error message: the fallback is the empty box
|
// Silent on failure. Nothing here is worth an error message: the fallback is the empty box
|
||||||
// the operator would otherwise have got, and an empty box is a working chat.
|
// the operator would otherwise have got, and an empty box is a working chat.
|
||||||
if (store && o.resume !== false) {
|
// The checkbox decides, and it defaults ON — an unset key means a first visit, not a refusal.
|
||||||
|
// Read here rather than from the element, because construction runs before the header is
|
||||||
|
// wired and an unchecked box must prevent the fetch, not undo it afterwards.
|
||||||
|
let _resumeWanted = true;
|
||||||
|
try { _resumeWanted = localStorage.getItem(RESUME_KEY) !== '0'; } catch (_) {}
|
||||||
|
const resumeEl = $('resume');
|
||||||
|
if (resumeEl) {
|
||||||
|
resumeEl.checked = _resumeWanted;
|
||||||
|
resumeEl.addEventListener('change', () => {
|
||||||
|
try { localStorage.setItem(RESUME_KEY, resumeEl.checked ? '1' : '0'); } catch (_) {}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (store && o.resume !== false && _resumeWanted) {
|
||||||
fetch(API + '?action=chats').then(r => r.json()).then(d => {
|
fetch(API + '?action=chats').then(r => r.json()).then(d => {
|
||||||
if (!d.ok || !(d.chats || []).length) return;
|
if (!d.ok || !(d.chats || []).length) return;
|
||||||
// Do not clobber a conversation the operator has already started. The list arrives after
|
// Do not clobber a conversation the operator has already started. The list arrives after
|
||||||
@@ -2073,9 +2140,17 @@ vv_ai_profiles_script();
|
|||||||
const pool = want ? d.chats.filter(c => (c.profile || 'chat') === want) : d.chats;
|
const pool = want ? d.chats.filter(c => (c.profile || 'chat') === want) : d.chats;
|
||||||
if (!pool.length) return;
|
if (!pool.length) return;
|
||||||
|
|
||||||
const last = pool.slice().sort((a, b) => (b.updated || 0) - (a.updated || 0))[0];
|
// This card's own last thread first, and only if it is still in the pool — deleted
|
||||||
|
// elsewhere, or moved to a profile this card does not show, both mean it is not ours to
|
||||||
|
// reopen. Newest-in-pool remains the answer for a first visit and for that case, which is
|
||||||
|
// what every card did unconditionally before.
|
||||||
|
let want_id = '';
|
||||||
|
try { want_id = localStorage.getItem(LAST_CHAT_KEY) || ''; } catch (_) {}
|
||||||
|
const mine = want_id ? pool.find(c => c.id === want_id) : null;
|
||||||
|
|
||||||
|
const last = mine || pool.slice().sort((a, b) => (b.updated || 0) - (a.updated || 0))[0];
|
||||||
if (last) loadChat(last.id);
|
if (last) loadChat(last.id);
|
||||||
}).catch(() => {});
|
}).catch(e => vvFetchErr('resume conversation', e));
|
||||||
}
|
}
|
||||||
|
|
||||||
const inst = {
|
const inst = {
|
||||||
@@ -2147,8 +2222,94 @@ vv_ai_profiles_script();
|
|||||||
if (!box) return null;
|
if (!box) return null;
|
||||||
let rows = [], activeId = '';
|
let rows = [], activeId = '';
|
||||||
|
|
||||||
|
// Optional first section: this card's own profile, collapsed, holding its most recent
|
||||||
|
// conversations. Underneath it the full list carries on unchanged.
|
||||||
|
//
|
||||||
|
// The two answer different questions. "Where was I in General Chat" is the one asked on a
|
||||||
|
// card pinned to General Chat, and a single flat list buries it under whatever the tabs have
|
||||||
|
// been doing — the profile with the most threads wins the top of the list regardless of which
|
||||||
|
// card you are looking at. Collapsed by default because the card resumes the newest of these
|
||||||
|
// anyway; opening it is for reaching the other nine.
|
||||||
|
const GROUPED = o.grouped !== false; // one collapsible section per profile
|
||||||
|
const GROUP_MAX = o.groupMax || 10; // per profile, and for the recent list below
|
||||||
|
const OPEN_KEY = 'vvAiListOpen:' + (o.into || '');
|
||||||
|
|
||||||
|
// Which sections are open, by profile key. Remembered as a set rather than one flag: the
|
||||||
|
// point of per-profile sections is that they are asked about independently.
|
||||||
|
let openSet = {};
|
||||||
|
try { openSet = JSON.parse(localStorage.getItem(OPEN_KEY) || '{}') || {}; } catch (_) { openSet = {}; }
|
||||||
|
// The card's own profile starts open — it is the one its operator is most likely to want,
|
||||||
|
// and on a card pinned to a profile an all-collapsed list says nothing on arrival.
|
||||||
|
if (o.groupProfile && !(o.groupProfile in openSet)) openSet[o.groupProfile] = true;
|
||||||
|
|
||||||
|
function rowHtml(c, P) {
|
||||||
|
const prof = (c.profile && c.profile !== 'chat' && P[c.profile]) ? P[c.profile].short : '';
|
||||||
|
const tag = [prof, c.scope || ''].filter(Boolean).join(' · ');
|
||||||
|
return `<div class="vv-ai-crow${c.id === activeId ? ' active' : ''}" data-id="${esc(c.id)}">`
|
||||||
|
+ `<span class="vv-ai-crow-t" title="${esc((tag ? tag + ' — ' : '') + c.title
|
||||||
|
+ ' · ' + chatAges(c))}">`
|
||||||
|
+ (tag ? `<span class="vv-ai-crow-s">${esc(tag)}</span> ` : '')
|
||||||
|
+ `${esc(c.title)}</span>`
|
||||||
|
+ `<span class="vv-ai-crow-m">${esc(ago(c.ts))}</span>`
|
||||||
|
+ `<span class="vv-ai-crow-x" data-del="${esc(c.id)}" title="Delete">×</span>`
|
||||||
|
+ `</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// One collapsible section per profile, each holding that profile's most recent GROUP_MAX.
|
||||||
|
//
|
||||||
|
// Profile order comes from the registry, not from the data, so the sections stay in the same
|
||||||
|
// places as threads come and go — a list that reorders itself under the cursor is a list you
|
||||||
|
// have to re-read every time. Profiles with nothing in them are still shown, with a count of
|
||||||
|
// zero: their absence would otherwise read as a bug on the day you first look for one.
|
||||||
|
function groupHtml() {
|
||||||
|
if (!GROUPED) return '';
|
||||||
|
const P = window.VvAiProfiles || {};
|
||||||
|
const keys = Object.keys(P);
|
||||||
|
if (!keys.length) return '';
|
||||||
|
|
||||||
|
const byProf = {};
|
||||||
|
for (const c of rows) (byProf[c.profile || 'chat'] = byProf[c.profile || 'chat'] || []).push(c);
|
||||||
|
|
||||||
|
let html = '';
|
||||||
|
for (const k of keys) {
|
||||||
|
const name = (P[k] && P[k].short) || k;
|
||||||
|
const mine = (byProf[k] || [])
|
||||||
|
.slice().sort((a, b) => (b.updated || b.ts || 0) - (a.updated || a.ts || 0))
|
||||||
|
.slice(0, GROUP_MAX);
|
||||||
|
const open = !!openSet[k];
|
||||||
|
html += `<div class="vv-ai-cgroup">`
|
||||||
|
+ `<div class="vv-ai-chead" data-grp="${esc(k)}" style="display:flex;align-items:center;
|
||||||
|
gap:6px;cursor:pointer;padding:3px 4px;user-select:none;">`
|
||||||
|
+ `<span style="color:#666;font-size:10px;width:8px;">${open ? '▾' : '▸'}</span>`
|
||||||
|
+ `<span style="color:#888;font-size:10px;font-weight:600;">${esc(name)}</span>`
|
||||||
|
+ `<span style="color:#333;font-size:9px;">${mine.length}</span>`
|
||||||
|
+ `</div>`
|
||||||
|
+ (open
|
||||||
|
? `<div class="vv-ai-clist">`
|
||||||
|
+ (mine.length ? mine.map(c => rowHtml(c, P)).join('')
|
||||||
|
: '<div class="vv-ai-none">none yet</div>')
|
||||||
|
+ `</div>`
|
||||||
|
: '')
|
||||||
|
+ `</div>`;
|
||||||
|
}
|
||||||
|
return html;
|
||||||
|
}
|
||||||
|
|
||||||
|
// The banner is a heading, not decoration: without it the rows below read as a sixth profile
|
||||||
|
// section rather than as the cross-profile recents.
|
||||||
|
function recentBanner(n) {
|
||||||
|
return `<div style="display:flex;align-items:center;gap:8px;margin:8px 0 4px;">`
|
||||||
|
+ `<div style="flex:1;border-bottom:1px solid #1e1e1e;"></div>`
|
||||||
|
+ `<span style="color:#4a4a4a;font-size:9px;text-transform:uppercase;
|
||||||
|
letter-spacing:.06em;white-space:nowrap;">Last ${n} conversations</span>`
|
||||||
|
+ `<div style="flex:1;border-bottom:1px solid #1e1e1e;"></div>`
|
||||||
|
+ `</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
function render() {
|
function render() {
|
||||||
if (!rows.length) {
|
if (!rows.length) {
|
||||||
|
// No sections on a genuinely empty store — five headings all reading 0 is a shape that
|
||||||
|
// implies something is filtered out rather than that nothing has been said yet.
|
||||||
box.innerHTML = '<div class="vv-ai-none">no saved conversations yet</div>';
|
box.innerHTML = '<div class="vv-ai-none">no saved conversations yet</div>';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -2160,27 +2321,32 @@ vv_ai_profiles_script();
|
|||||||
// makes the rows that are genuinely different harder to pick out, which is the opposite
|
// makes the rows that are genuinely different harder to pick out, which is the opposite
|
||||||
// of the point.
|
// of the point.
|
||||||
const P = window.VvAiProfiles || {};
|
const P = window.VvAiProfiles || {};
|
||||||
box.innerHTML = '<div class="vv-ai-clist">' + rows.map(c => {
|
// Capped to the same GROUP_MAX. Uncapped, this repeated most of what the sections above
|
||||||
const prof = (c.profile && c.profile !== 'chat' && P[c.profile]) ? P[c.profile].short : '';
|
// already show and pushed them off the top of a card that is only 300px tall.
|
||||||
const tag = [prof, c.scope || ''].filter(Boolean).join(' · ');
|
const recent = rows.slice()
|
||||||
return `<div class="vv-ai-crow${c.id === activeId ? ' active' : ''}" data-id="${esc(c.id)}">`
|
.sort((a, b) => (b.updated || b.ts || 0) - (a.updated || a.ts || 0))
|
||||||
+ `<span class="vv-ai-crow-t" title="${esc((tag ? tag + ' — ' : '') + c.title
|
.slice(0, GROUP_MAX);
|
||||||
+ ' · ' + chatAges(c))}">`
|
box.innerHTML = groupHtml()
|
||||||
+ (tag ? `<span class="vv-ai-crow-s">${esc(tag)}</span> ` : '')
|
+ (GROUPED ? recentBanner(recent.length) : '')
|
||||||
+ `${esc(c.title)}</span>`
|
+ '<div class="vv-ai-clist">' + recent.map(c => rowHtml(c, P)).join('') + '</div>';
|
||||||
+ `<span class="vv-ai-crow-m">${esc(ago(c.ts))}</span>`
|
|
||||||
+ `<span class="vv-ai-crow-x" data-del="${esc(c.id)}" title="Delete">×</span>`
|
|
||||||
+ `</div>`;
|
|
||||||
}).join('') + '</div>';
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function load() {
|
function load() {
|
||||||
fetch(API + '?action=chats').then(r => r.json())
|
fetch(API + '?action=chats').then(r => r.json())
|
||||||
.then(d => { if (d.ok) { rows = d.chats || []; render(); } })
|
.then(d => { if (d.ok) { rows = d.chats || []; render(); } })
|
||||||
.catch(() => {});
|
.catch(e => vvFetchErr('conversation list', e));
|
||||||
}
|
}
|
||||||
|
|
||||||
box.addEventListener('click', e => {
|
box.addEventListener('click', e => {
|
||||||
|
// Before the row handler: the header sits above the rows and must not be read as one.
|
||||||
|
const grp = e.target.closest('[data-grp]');
|
||||||
|
if (grp) {
|
||||||
|
const k = grp.dataset.grp;
|
||||||
|
openSet[k] = !openSet[k];
|
||||||
|
try { localStorage.setItem(OPEN_KEY, JSON.stringify(openSet)); } catch (_) {}
|
||||||
|
render();
|
||||||
|
return;
|
||||||
|
}
|
||||||
const del = e.target.closest('[data-del]');
|
const del = e.target.closest('[data-del]');
|
||||||
if (del) {
|
if (del) {
|
||||||
e.stopPropagation();
|
e.stopPropagation();
|
||||||
@@ -2191,7 +2357,7 @@ vv_ai_profiles_script();
|
|||||||
// from the store, which would silently resurrect it on the next turn.
|
// from the store, which would silently resurrect it on the next turn.
|
||||||
if (del.dataset.del === activeId && o.chat) o.chat.newChat();
|
if (del.dataset.del === activeId && o.chat) o.chat.newChat();
|
||||||
load();
|
load();
|
||||||
}).catch(() => {});
|
}).catch(e => vvFetchErr('delete conversation', e));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const row = e.target.closest('.vv-ai-crow');
|
const row = e.target.closest('.vv-ai-crow');
|
||||||
@@ -2216,7 +2382,8 @@ vv_ai_profiles_script();
|
|||||||
// broke the tab it lives on.
|
// broke the tab it lives on.
|
||||||
function vv_ai_chat_keys(): array {
|
function vv_ai_chat_keys(): array {
|
||||||
return [
|
return [
|
||||||
['keys' => ['Ctrl', 'Enter'], 'what' => 'Ask', 'id' => 'send'],
|
['keys' => ['Enter'], 'what' => 'Ask', 'id' => 'send'],
|
||||||
|
['keys' => ['Ctrl', 'Enter'], 'what' => 'New line', 'id' => 'nl'],
|
||||||
['keys' => ['↑'], 'what' => 'Your last question, to edit', 'id' => 'recall'],
|
['keys' => ['↑'], 'what' => 'Your last question, to edit', 'id' => 'recall'],
|
||||||
['keys' => ['/'], 'what' => 'Command — /help lists them', 'id' => 'slash'],
|
['keys' => ['/'], 'what' => 'Command — /help lists them', 'id' => 'slash'],
|
||||||
['keys' => ['Esc'], 'what' => 'Close a menu, or clear the box', 'id' => 'esc'],
|
['keys' => ['Esc'], 'what' => 'Close a menu, or clear the box', 'id' => 'esc'],
|
||||||
@@ -2293,7 +2460,21 @@ function vv_ai_chat_markup(string $prefix, array $o = []): void {
|
|||||||
<div class="vv-ai-head">
|
<div class="vv-ai-head">
|
||||||
<span class="vv-ai-head-t">
|
<span class="vv-ai-head-t">
|
||||||
<?php if (!empty($o['icon'])): ?><span class="vv-ico"><?= $o['icon'] ?></span><?php endif; ?>
|
<?php if (!empty($o['icon'])): ?><span class="vv-ico"><?= $o['icon'] ?></span><?php endif; ?>
|
||||||
<?= htmlspecialchars($o['title'] ?? 'Assistant') ?>
|
<span id="<?= $p ?>-title"><?= htmlspecialchars($o['title'] ?? 'Assistant') ?></span>
|
||||||
|
<?php if (!empty($o['mesh'])): ?>
|
||||||
|
<!-- The badge is the whole reason a toggle is acceptable here. Behind a switch, an arriving
|
||||||
|
message is invisible until somebody happens to flip it — and the messages this carries
|
||||||
|
are the ones you most need to not miss ("mine is going down for a week"). Purple because
|
||||||
|
nothing else on these pages is, so it reads as new rather than as another warning. -->
|
||||||
|
<button type="button" id="<?= $p ?>-mode" class="vv-ai-btn ghost"
|
||||||
|
onclick="vvNcMode()" title="Switch between the assistant and the mesh"
|
||||||
|
style="margin-left:8px;font-size:10px;padding:2px 8px;">
|
||||||
|
<span id="<?= $p ?>-mode-l">Mesh Chat</span>
|
||||||
|
<span id="vv-nc-badge" hidden
|
||||||
|
style="margin-left:5px;background:#7c4dff;color:#fff;border-radius:8px;
|
||||||
|
padding:0 6px;font-size:9px;font-weight:700;">✉ 0</span>
|
||||||
|
</button>
|
||||||
|
<?php endif; ?>
|
||||||
</span>
|
</span>
|
||||||
|
|
||||||
<!-- Centred by giving the title and the right-hand group equal flex, not by margins — the
|
<!-- Centred by giving the title and the right-hand group equal flex, not by margins — the
|
||||||
@@ -2303,15 +2484,42 @@ function vv_ai_chat_markup(string $prefix, array $o = []): void {
|
|||||||
Both are view controls: neither changes what is asked, only what you watch while it
|
Both are view controls: neither changes what is asked, only what you watch while it
|
||||||
happens. That is why they sit here with the window controls rather than in the composer
|
happens. That is why they sit here with the window controls rather than in the composer
|
||||||
beside the things that shape the question. -->
|
beside the things that shape the question. -->
|
||||||
<span class="vv-ai-head-c">
|
<span class="vv-ai-head-c" id="<?= $p ?>-cb-ai">
|
||||||
<label class="vv-ai-cb" title="Show the model's reasoning as it is written, not after">
|
<label class="vv-ai-cb" title="Show the model's reasoning as it is written, not after">
|
||||||
<input type="checkbox" id="<?= $p ?>-see-think"> <span>Reasoning</span>
|
<input type="checkbox" id="<?= $p ?>-see-think"> <span>Reasoning</span>
|
||||||
</label>
|
</label>
|
||||||
|
<!-- Governs the NEXT open, not this one: by the time it can be clicked the thread it would
|
||||||
|
have resumed is already on screen. The title says so, because a checkbox that appears
|
||||||
|
to do nothing when ticked is worse than no checkbox. -->
|
||||||
|
<label class="vv-ai-cb" title="Reopen this card on your last conversation. Applies next time the page loads.">
|
||||||
|
<input type="checkbox" id="<?= $p ?>-resume" checked> <span>Load Last</span>
|
||||||
|
</label>
|
||||||
<label class="vv-ai-cb" title="Follow the newest line. Unticks when you scroll up, re-ticks at the bottom.">
|
<label class="vv-ai-cb" title="Follow the newest line. Unticks when you scroll up, re-ticks at the bottom.">
|
||||||
<input type="checkbox" id="<?= $p ?>-follow" checked> <span>Auto Scroll</span>
|
<input type="checkbox" id="<?= $p ?>-follow" checked> <span>Auto Scroll</span>
|
||||||
</label>
|
</label>
|
||||||
</span>
|
</span>
|
||||||
|
|
||||||
|
<?php if (!empty($o['mesh'])): ?>
|
||||||
|
<!-- The mesh's three, in the same place and the same three slots. Switching mode swaps this
|
||||||
|
group for the one above rather than leaving the assistant's controls sitting over a
|
||||||
|
conversation they do not govern — which is what a second row inside the pane amounted to,
|
||||||
|
and why the banner looked unchanged after switching. -->
|
||||||
|
<span class="vv-ai-head-c" id="<?= $p ?>-cb-mesh" hidden>
|
||||||
|
<label class="vv-ai-cb"
|
||||||
|
title="Render colours, fonts and sizes as sent. Unticked shows every message as plain text.">
|
||||||
|
<input type="checkbox" id="vv-nc-fmt-on" checked onchange="vvNcPref();vvNcRender()"> <span>Formatting</span>
|
||||||
|
</label>
|
||||||
|
<label class="vv-ai-cb"
|
||||||
|
title="Show only what has arrived since you last looked. Unticked shows the whole conversation.">
|
||||||
|
<input type="checkbox" id="vv-nc-unread" onchange="vvNcRender()"> <span>Only New</span>
|
||||||
|
</label>
|
||||||
|
<label class="vv-ai-cb"
|
||||||
|
title="Follow the newest message. Unticks when you scroll up, re-ticks at the bottom.">
|
||||||
|
<input type="checkbox" id="vv-nc-follow" checked onchange="vvNcPref()"> <span>Auto Scroll</span>
|
||||||
|
</label>
|
||||||
|
</span>
|
||||||
|
<?php endif; ?>
|
||||||
|
|
||||||
<span class="vv-ai-head-r">
|
<span class="vv-ai-head-r">
|
||||||
<button class="vv-ai-btn ghost vv-ai-size" id="<?= $p ?>-size" type="button"
|
<button class="vv-ai-btn ghost vv-ai-size" id="<?= $p ?>-size" type="button"
|
||||||
style="display:none">Medium</button>
|
style="display:none">Medium</button>
|
||||||
@@ -2347,6 +2555,9 @@ function vv_ai_chat_markup(string $prefix, array $o = []): void {
|
|||||||
|
|
||||||
<!-- Positioned wrapper so the jump pill can sit over the transcript's bottom edge without
|
<!-- Positioned wrapper so the jump pill can sit over the transcript's bottom edge without
|
||||||
being clipped by its overflow, and without taking a row of its own when hidden. -->
|
being clipped by its overflow, and without taking a row of its own when hidden. -->
|
||||||
|
<!-- Two panes, one card. The mesh pane is a sibling of this one rather than a second card, so
|
||||||
|
switching what the card is does not give the page a second header to look at. -->
|
||||||
|
<div id="<?= $p ?>-pane-ai">
|
||||||
<div class="vv-ai-chat-wrap">
|
<div class="vv-ai-chat-wrap">
|
||||||
<div class="vv-ai-chat" id="<?= $p ?>-chat"<?= $style ?>>
|
<div class="vv-ai-chat" id="<?= $p ?>-chat"<?= $style ?>>
|
||||||
<div class="vv-ai-empty"><?= htmlspecialchars($empty) ?></div>
|
<div class="vv-ai-empty"><?= htmlspecialchars($empty) ?></div>
|
||||||
@@ -2416,6 +2627,8 @@ function vv_ai_chat_markup(string $prefix, array $o = []): void {
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
</div><?php /* close -pane-ai */ ?>
|
||||||
|
<?php if (!empty($o['mesh']) && function_exists('vv_nc_pane_markup')) vv_nc_pane_markup($p, !empty($o['meshDefault']), $height, $tall); ?>
|
||||||
</div>
|
</div>
|
||||||
<?php
|
<?php
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,31 +8,53 @@
|
|||||||
// the prompt on its own. A proposal is filed; the operator accepts or dismisses it; accepted
|
// the prompt on its own. A proposal is filed; the operator accepts or dismisses it; accepted
|
||||||
// text lands in the learned memory slot, which the prompt explicitly ranks BELOW retrieval.
|
// text lands in the learned memory slot, which the prompt explicitly ranks BELOW retrieval.
|
||||||
//
|
//
|
||||||
// WHY IT IS A PROPOSAL AND NOT A WRITE
|
// DESIGN PRINCIPLES
|
||||||
// Memory is injected into every future prompt. A model that writes its own memory writes its
|
// It is a proposal, never a write.
|
||||||
// own mistakes, and then reads them back as established fact — growing more confident on each
|
// Memory is injected into every future prompt. A model that writes its own memory writes
|
||||||
// turn while the actual source code says otherwise. The cost of a bad proposal has to be one
|
// its own mistakes, and then reads them back as established fact — growing more confident
|
||||||
// dismissal, not a permanently poisoned prompt. This is the same two-gate shape the repair
|
// on each turn while the actual source code says otherwise. The cost of a bad proposal has
|
||||||
// system uses, for the same reason: whether something should be remembered is intent, and a
|
// to be one dismissal, not a permanently poisoned prompt. This is the same two-gate shape
|
||||||
// model cannot prove intent.
|
// the repair system uses, for the same reason: whether something should be remembered is
|
||||||
|
// intent, and a model cannot prove intent.
|
||||||
//
|
//
|
||||||
// WHY DEDUP IS NOT THE MODEL'S JOB
|
// Dedup is deterministic, not the model's job.
|
||||||
// "Do I already know this" is a semantic comparison, and a 14B at IQ4_XS is confidently wrong
|
// "Do I already know this" is a semantic comparison, and a 14B at IQ4_XS is confidently
|
||||||
// at it often enough to matter — with every miss costing budget permanently. So dedup here is
|
// wrong at it often enough to matter — with every miss costing budget permanently. So
|
||||||
// deterministic: normalise, then reject on exact match or containment in either direction
|
// dedup here is deterministic: normalise, then reject on exact match or containment in
|
||||||
// against assisted memory, learned memory, and everything previously dismissed. It will let
|
// either direction against assisted memory, learned memory, and everything previously
|
||||||
// through a reworded duplicate; it will never silently drop something new, and that is the
|
// dismissed. It will let through a reworded duplicate; it will never silently drop
|
||||||
// right way round for a store the operator reviews anyway.
|
// something new, and that is the right way round for a store the operator reviews anyway.
|
||||||
//
|
//
|
||||||
// GATES
|
// OPERATIONAL SAFEGUARDS
|
||||||
// AI_MEMORY_LEARN_ENABLED false — nothing is proposed, and the prompt gains nothing
|
// Two gates, and the second cannot outrank the first.
|
||||||
// AI_MEMORY_LEARN_AUTO_ACCEPT false — accepted writes happen only when the operator says so
|
// AI_MEMORY_LEARN_ENABLED off means nothing is proposed at all, so auto-accept with
|
||||||
// The second cannot outrank the first: auto-accept with proposing off does nothing at all.
|
// proposing off does nothing. Neither defaults to on.
|
||||||
|
//
|
||||||
|
// Dismissed rows are kept, not deleted.
|
||||||
|
// They are the only thing that stops the same suggestion arriving again every night. A
|
||||||
|
// store that forgot its refusals would re-propose what the operator has already judged.
|
||||||
|
//
|
||||||
|
// Nothing here reaches the prompt directly. Accepted text lands in the learned memory slot,
|
||||||
|
// which the prompt ranks BELOW retrieval — so even an accepted mistake cannot outrank the
|
||||||
|
// source code it contradicts.
|
||||||
|
//
|
||||||
|
// EXPORTS
|
||||||
|
// Gates vv_ai_mem_learn_enabled(), vv_ai_mem_learn_auto()
|
||||||
|
// Dedup vv_ai_mem_norm(), vv_ai_mem_known(), vv_ai_mem_is_dup(), vv_ai_mem_is_ui_fact()
|
||||||
|
// Store vv_ai_mem_dir(), vv_ai_mem_propose(), vv_ai_mem_list(), vv_ai_mem_remove(),
|
||||||
|
// vv_ai_mem_append(), vv_ai_mem_write_row()
|
||||||
|
// Operator vv_ai_mem_action()
|
||||||
|
// — accept or dismiss one proposal. The only entry that changes what a future
|
||||||
|
// prompt will contain.
|
||||||
|
//
|
||||||
|
// CONFIGURATION
|
||||||
|
// master.conf
|
||||||
|
// AI_MEMORY_LEARN_ENABLED propose at all. Default false.
|
||||||
|
// AI_MEMORY_LEARN_AUTO_ACCEPT write accepted text without asking. Default false.
|
||||||
//
|
//
|
||||||
// STORE
|
// STORE
|
||||||
// data/ai/mem_proposals/<id>.json — one file per proposal, mirroring the findings store.
|
// data/ai/mem_proposals/<id>.json — one file per proposal, mirroring the findings store.
|
||||||
// States: open | accepted | dismissed. Dismissed rows are KEPT, because they are what stops
|
// States: open | accepted | dismissed.
|
||||||
// the same suggestion arriving again every night.
|
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
require_once __DIR__ . '/ai.php';
|
require_once __DIR__ . '/ai.php';
|
||||||
@@ -194,7 +216,7 @@ function vv_ai_mem_append(string $text): array {
|
|||||||
// Takes one line back out of the learned slot. The counterpart to vv_ai_mem_append(), and the
|
// Takes one line back out of the learned slot. The counterpart to vv_ai_mem_append(), and the
|
||||||
// reason a decided proposal can be changed at all: without this, "accepted" was a one-way door
|
// reason a decided proposal can be changed at all: without this, "accepted" was a one-way door
|
||||||
// and the only way out was editing mem_learned.md by hand. An accepted line joins every future
|
// and the only way out was editing mem_learned.md by hand. An accepted line joins every future
|
||||||
// prompt, and a fact that is true today — HOST2 is offline — becomes actively wrong the day it
|
// prompt, and a fact that is true today — a host is unreachable — becomes actively wrong the day it
|
||||||
// stops being true, with nothing to expire it.
|
// stops being true, with nothing to expire it.
|
||||||
//
|
//
|
||||||
// Matched on the line this file wrote: "- <text> (<date>)". A line the operator has since edited
|
// Matched on the line this file wrote: "- <text> (<date>)". A line the operator has since edited
|
||||||
|
|||||||
@@ -5,47 +5,51 @@
|
|||||||
// shared chat include and every page that renders it — reads it from here instead of
|
// shared chat include and every page that renders it — reads it from here instead of
|
||||||
// restating it.
|
// restating it.
|
||||||
//
|
//
|
||||||
// WHY THIS EXISTS
|
// DESIGN PRINCIPLES
|
||||||
// A profile used to be defined in five places: history depth in api/ai.php, capabilities in
|
// One definition, not five.
|
||||||
// include/ai.php, label/hint/depth again in the chat's JavaScript, a prompt branch in the
|
// A profile used to be defined in five places: history depth in api/ai.php, capabilities
|
||||||
// worker, and a label map in pages/scheduler.php. They had already drifted — the JavaScript
|
// in include/ai.php, label/hint/depth again in the chat's JavaScript, a prompt branch in
|
||||||
// knew three profiles where PHP knew four, so the shared chat could not offer troubleshoot at
|
// the worker, and a label map in pages/scheduler.php. They had already drifted — the
|
||||||
// all and the Scheduler dock hand-rolled its own labels to compensate. The include carried a
|
// JavaScript knew three profiles where PHP knew four, so the shared chat could not offer
|
||||||
// comment telling the next person not to let the two tables diverge, which is a comment doing
|
// troubleshoot at all and the Scheduler dock hand-rolled its own labels to compensate. The
|
||||||
// a data structure's job.
|
// include carried a comment telling the next person not to let the two tables diverge,
|
||||||
|
// which is a comment doing a data structure's job.
|
||||||
//
|
//
|
||||||
// WHAT LIVES HERE, AND WHAT DELIBERATELY DOES NOT
|
// What lives here is whatever more than one file must agree on.
|
||||||
// Here: anything more than one file needs to agree on — the set of profiles, their labels and
|
// The set of profiles, their labels and hints, history depth, capabilities, and whether a
|
||||||
// hints, history depth, capabilities, and whether a profile is offered as a button.
|
// profile is offered as a button.
|
||||||
//
|
//
|
||||||
// Not here: the system prompts. They are long, delicate, and have exactly one consumer, so
|
// Not the system prompts. They are long, delicate, and have exactly one consumer, so
|
||||||
// moving them would be churn against the most sensitive text in the subsystem for no reduction
|
// moving them would be churn against the most sensitive text in the subsystem for no
|
||||||
// in duplication. Tools/ai_chat_worker.php still owns them; it just keys off ids validated
|
// reduction in duplication. Tools/ai_chat_worker.php still owns them; it just keys off ids
|
||||||
// here rather than an if-chain that invents its own vocabulary.
|
// validated here rather than an if-chain that invents its own vocabulary.
|
||||||
//
|
//
|
||||||
// CAPABILITIES ARE PER PROFILE, NOT PER CAPABILITY
|
// Capabilities are stored per profile, not per capability.
|
||||||
// The old table was inverted — capability => [profiles] — which reads well when adding a
|
// The old table was inverted — capability => [profiles] — which reads well when adding a
|
||||||
// capability and badly when answering the question actually asked at runtime, which is always
|
// capability and badly when answering the question actually asked at runtime, which is
|
||||||
// "what can this profile do". Same content, turned the right way round.
|
// always "what can this profile do". Same content, turned the right way round.
|
||||||
//
|
//
|
||||||
// A profile is a contract plus a set of inputs, and the inputs are the half that has to be
|
// A profile is a contract plus a set of inputs, and the inputs are the half that has to be
|
||||||
// enforced rather than requested. The caps list is that half.
|
// enforced rather than requested. The caps list is that half.
|
||||||
//
|
//
|
||||||
// It exists because the alternative already failed. The same permissions used to live as a
|
// OPERATIONAL SAFEGUARDS
|
||||||
// dozen `$profile === 'varaverk' || $profile === 'troubleshoot'` conditions spread across the
|
// The capability list is the grant, and it is enforced in one place.
|
||||||
// worker, and answering "may chat ever be shown a log?" meant reading all of them. It could —
|
// The same permissions used to live as a dozen `$profile === 'varaverk' || $profile ===
|
||||||
// a gate added for run-outcome questions granted it by omission, and the chat profile, whose
|
// 'troubleshoot'` conditions spread across the worker, and answering "may chat ever be
|
||||||
// entire value is that it has NOT been shown this installation, was one phrasing away from
|
// shown a log?" meant reading all of them. It could — a gate added for run-outcome
|
||||||
// being handed a health sweep and 120 lines of log. Nothing about that was visible at the
|
// questions granted it by omission, and the chat profile, whose entire value is that it
|
||||||
// point of the mistake. Here it would have been one missing word on one line.
|
// has NOT been shown this installation, was one phrasing away from being handed a health
|
||||||
|
// sweep and 120 lines of log. Nothing about that was visible at the point of the mistake.
|
||||||
|
// Here it would have been one missing word on one line.
|
||||||
//
|
//
|
||||||
// A capability is permission, not need. varaverk holds 'health' but only attaches it when the
|
// A capability is permission, not need.
|
||||||
// question looks diagnostic; troubleshoot attaches it always. The gates decide whether an
|
// varaverk holds 'health' but only attaches it when the question looks diagnostic;
|
||||||
// input is warranted, this decides whether it is allowed, and a gate can never widen the grant.
|
// troubleshoot attaches it always. The gates decide whether an input is warranted, this
|
||||||
|
// decides whether it is allowed, and a gate can never widen the grant.
|
||||||
//
|
//
|
||||||
// chat holding an empty capability list is a guarantee, not an oversight. Anything added to it
|
// chat holding an empty capability list is a guarantee, not an oversight.
|
||||||
// stops being general chat and becomes an assistant that sometimes lies about this
|
// Anything added to it stops being general chat and becomes an assistant that sometimes
|
||||||
// installation.
|
// lies about this installation.
|
||||||
//
|
//
|
||||||
// EXPORTS
|
// EXPORTS
|
||||||
// vv_ai_profiles() the whole table
|
// vv_ai_profiles() the whole table
|
||||||
@@ -116,7 +120,7 @@ const VV_AI_PROFILES_DEF = [
|
|||||||
// system_state is read-only and shared with repair. Both need to know a container is down
|
// system_state is read-only and shared with repair. Both need to know a container is down
|
||||||
// or a pool is full to explain anything about this machine rather than about Unraid in
|
// or a pool is full to explain anything about this machine rather than about Unraid in
|
||||||
// general; neither gets a way to act on it, and only repair can change a setting.
|
// general; neither gets a way to act on it, and only repair can change a setting.
|
||||||
'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log',
|
'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log',
|
||||||
'incidents', 'conf_lookup', 'file_bugs'],
|
'incidents', 'conf_lookup', 'file_bugs'],
|
||||||
],
|
],
|
||||||
// The only profile that may change a setting, and the only one not offered as a button.
|
// The only profile that may change a setting, and the only one not offered as a button.
|
||||||
@@ -136,7 +140,7 @@ const VV_AI_PROFILES_DEF = [
|
|||||||
'hint' => 'Works through a finding with you, and can apply a fix you approve.',
|
'hint' => 'Works through a finding with you, and can apply a fix you approve.',
|
||||||
'turns' => 3,
|
'turns' => 3,
|
||||||
'ui' => false,
|
'ui' => false,
|
||||||
'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log', 'incidents',
|
'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log', 'incidents',
|
||||||
'conf_lookup', 'conf_write', 'probe', 'file_findings', 'phrasebook', 'past_fixes'],
|
'conf_lookup', 'conf_write', 'probe', 'file_findings', 'phrasebook', 'past_fixes'],
|
||||||
],
|
],
|
||||||
];
|
];
|
||||||
@@ -148,6 +152,7 @@ const VV_AI_CAP_MEANING = [
|
|||||||
'kind_filter' => 'the retrieval kind filter the page exposes',
|
'kind_filter' => 'the retrieval kind filter the page exposes',
|
||||||
'health' => 'live health sweep measured at question time — the AI subsystem only',
|
'health' => 'live health sweep measured at question time — the AI subsystem only',
|
||||||
'system_state' => 'read-only view of the machine: hardware, containers, pools, array, UPS',
|
'system_state' => 'read-only view of the machine: hardware, containers, pools, array, UPS',
|
||||||
|
'fallback_state' => 'whether a failover would actually work: state, tiers, and whether the partner really has the covered containers',
|
||||||
'run_evidence' => 'run record and log tail for a script named in the question',
|
'run_evidence' => 'run record and log tail for a script named in the question',
|
||||||
'scoped_log' => 'log tail for whatever the operator currently has open',
|
'scoped_log' => 'log tail for whatever the operator currently has open',
|
||||||
'incidents' => 'operator-written history of what previously went wrong with this thing',
|
'incidents' => 'operator-written history of what previously went wrong with this thing',
|
||||||
|
|||||||
@@ -310,9 +310,25 @@ function vv_ai_finding_get(string $id): ?array {
|
|||||||
|
|
||||||
// Files a finding, or increments the one already describing this fault.
|
// Files a finding, or increments the one already describing this fault.
|
||||||
//
|
//
|
||||||
|
// Sweeps run on every node — each reads its own logs, which is the only place they exist — but the
|
||||||
|
// store is the AI owner's, so the operator answers one list instead of one per machine. A mirror
|
||||||
|
// therefore forwards; the owner writes. The split is here rather than in the sweep so that every
|
||||||
|
// caller files a finding the same way and none of them has to know where the store lives.
|
||||||
|
//
|
||||||
// $f expects: kind, subject, conf_key, conf_file, observed, evidence, source_log
|
// $f expects: kind, subject, conf_key, conf_file, observed, evidence, source_log
|
||||||
// and optionally: proposed, proven, state, note
|
// and optionally: proposed, proven, state, note, host
|
||||||
function vv_ai_finding_write(array $f): array {
|
function vv_ai_finding_write(array $f): array {
|
||||||
|
if (vv_ai_is_owner()) return vv_ai_finding_write_local($f);
|
||||||
|
|
||||||
|
require_once __DIR__ . '/ai_rpc.php';
|
||||||
|
$status = 200;
|
||||||
|
$r = vv_ai_rpc('finding_write', ['finding' => json_encode($f)], true, $status);
|
||||||
|
// Not swallowed. A sweep that cannot reach the owner has found something and failed to record
|
||||||
|
// it, and a caller told "ok" would move on and never retry.
|
||||||
|
return is_array($r) ? $r : ['ok' => false, 'error' => 'finding_write: no response from the AI owner'];
|
||||||
|
}
|
||||||
|
|
||||||
|
function vv_ai_finding_write_local(array $f): array {
|
||||||
$kind = (string)($f['kind'] ?? '');
|
$kind = (string)($f['kind'] ?? '');
|
||||||
$subject = trim((string)($f['subject'] ?? ''));
|
$subject = trim((string)($f['subject'] ?? ''));
|
||||||
$confKey = trim((string)($f['conf_key'] ?? ''));
|
$confKey = trim((string)($f['conf_key'] ?? ''));
|
||||||
@@ -337,13 +353,18 @@ function vv_ai_finding_write(array $f): array {
|
|||||||
if (!isset(VV_AI_FINDING_STATES[$state])) $state = 'open';
|
if (!isset(VV_AI_FINDING_STATES[$state])) $state = 'open';
|
||||||
|
|
||||||
$now = time();
|
$now = time();
|
||||||
$id = vv_ai_finding_id($kind, $subject, $ref);
|
// Which machine this finding is about, resolved once and used for both the hash and the
|
||||||
|
// record. $f['host'] was accepted by the array below but the id was always hashed locally,
|
||||||
|
// so a collected partner finding hashed as ours — the exact collision the host-in-the-hash
|
||||||
|
// comment above exists to prevent.
|
||||||
|
$host = trim((string)($f['host'] ?? '')) ?: vv_detect_host();
|
||||||
|
$id = vv_ai_finding_id($kind, $subject, $ref, $host);
|
||||||
$rec = [
|
$rec = [
|
||||||
'id' => $id,
|
'id' => $id,
|
||||||
// Which machine this is about. Written even on a single-host install, because the store
|
// Which machine this is about. Written even on a single-host install, because the store
|
||||||
// outlives the topology — a finding filed today is still on disk when the second node
|
// outlives the topology — a finding filed today is still on disk when the second node
|
||||||
// arrives, and one without a host is a record nobody can place.
|
// arrives, and one without a host is a record nobody can place.
|
||||||
'host' => (string)($f['host'] ?? vv_detect_host()),
|
'host' => $host,
|
||||||
'kind' => $kind,
|
'kind' => $kind,
|
||||||
'subject' => mb_substr($subject, 0, 120),
|
'subject' => mb_substr($subject, 0, 120),
|
||||||
'conf_key' => $confKey,
|
'conf_key' => $confKey,
|
||||||
@@ -369,7 +390,6 @@ function vv_ai_finding_write(array $f): array {
|
|||||||
'severity' => vv_ai_finding_severity(['kind' => $kind, 'conf_key' => $confKey,
|
'severity' => vv_ai_finding_severity(['kind' => $kind, 'conf_key' => $confKey,
|
||||||
'arr_type' => (string)($f['arr_type'] ?? ''),
|
'arr_type' => (string)($f['arr_type'] ?? ''),
|
||||||
'sys_level' => (string)($f['sys_level'] ?? '')]),
|
'sys_level' => (string)($f['sys_level'] ?? '')]),
|
||||||
'host' => vv_detect_host(),
|
|
||||||
'first' => $now,
|
'first' => $now,
|
||||||
'last' => $now,
|
'last' => $now,
|
||||||
'seen' => 1,
|
'seen' => 1,
|
||||||
|
|||||||
@@ -0,0 +1,197 @@
|
|||||||
|
<?php
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// PURPOSE
|
||||||
|
// Where an AI action runs, and how it gets there. The mesh shares one AI: the owner node holds
|
||||||
|
// the model, the index and the shared memory, and every other node reaches them over SSH rather
|
||||||
|
// than keeping a second copy of any of it.
|
||||||
|
//
|
||||||
|
// Two exports. vv_ai_route() answers "local, remote, or refused" for one action on this node;
|
||||||
|
// vv_ai_rpc() carries a remote one to the owner and brings back its answer verbatim.
|
||||||
|
//
|
||||||
|
// OPERATIONAL MODEL
|
||||||
|
// Three routes, decided per action rather than per host:
|
||||||
|
//
|
||||||
|
// LOCAL this node answers. Everything on the owner. On a mirror, the chat store only —
|
||||||
|
// conversations are per-node by design, so they never leave the box they were had on.
|
||||||
|
// REMOTE forwarded to the owner: generation, retrieval, stats, the shared memory, findings.
|
||||||
|
// DENY refused with a 404. The curated writes — memory and bug reports — are the owner's.
|
||||||
|
//
|
||||||
|
// Transport is SSH over the trust partnership_onboard.sh already establishes, the same as
|
||||||
|
// node_chat and conf_sync: no listener, no new port, Tailscale-only for free. The request is
|
||||||
|
// JSON on stdin, the response is JSON on stdout, and Tools/ai_rpc.php on the far side hands both
|
||||||
|
// to the same vv_ai_dispatch() this node would have called locally.
|
||||||
|
//
|
||||||
|
// DESIGN PRINCIPLES
|
||||||
|
// Share, not copy.
|
||||||
|
// A mirror does not hold the index, the memory or the model and does not sync them. There
|
||||||
|
// is one of each, on the owner, and the mesh asks it. Copies of curated, hand-edited state
|
||||||
|
// are copies that can disagree, and reconciling them needs tombstones and an offline story
|
||||||
|
// — the same complexity node_chat's local-only delete deliberately refused.
|
||||||
|
//
|
||||||
|
// The job lives where the model lives.
|
||||||
|
// ask returns the owner's token and poll asks the owner about it, so the token-and-poll
|
||||||
|
// contract is unchanged; it simply resolves on another box. Nothing about the page changes.
|
||||||
|
//
|
||||||
|
// Chats stay home, memory is shared.
|
||||||
|
// A conversation is this operator's, on this node. What the assistant *knows* — the memory
|
||||||
|
// file, the learned notes, the phrasebook — is the owner's and is shared by everyone. The
|
||||||
|
// history for a turn travels in the request, so where chats are stored is independent of
|
||||||
|
// where generation happens.
|
||||||
|
//
|
||||||
|
// OPERATIONAL SAFEGUARDS
|
||||||
|
// The remote path is the WebGUI symlink, not a discovered one.
|
||||||
|
// /usr/local/emhttp/plugins/varaverk is what Unraid serves the plugin from on every node,
|
||||||
|
// whatever storage mode it uses. node_chat reads the partner's varaverk.cfg first because a
|
||||||
|
// delivery is occasional; poll runs once a second per open tab and cannot afford a second
|
||||||
|
// SSH round trip to find a path. If this symlink is wrong the whole plugin is already
|
||||||
|
// broken on that node, so it is not a weaker assumption than the one it replaces.
|
||||||
|
//
|
||||||
|
// One multiplexed connection, not one per call.
|
||||||
|
// ControlMaster with ControlPersist, socket in tmpfs. A fresh SSH handshake is 100-300ms;
|
||||||
|
// paying it per poll, per open tab, would make the assistant feel broken on a mirror.
|
||||||
|
//
|
||||||
|
// A transport failure is named, never rendered as an empty success.
|
||||||
|
// Unreachable owner, missing shim and unparseable output are three different errors and
|
||||||
|
// each says so. An empty banner that looks like "nothing to report" is the failure mode
|
||||||
|
// worth spending three messages on.
|
||||||
|
//
|
||||||
|
// Nothing here decides trust.
|
||||||
|
// Possession of the partnership SSH key is the authorization, established at onboard. This
|
||||||
|
// file routes; it does not authenticate.
|
||||||
|
//
|
||||||
|
// EXPORTS
|
||||||
|
// VV_AI_ROUTE_LOCAL / _REMOTE / _DENY
|
||||||
|
// vv_ai_route() action → route for this node
|
||||||
|
// vv_ai_rpc() forward one action to the owner, return its response body
|
||||||
|
//
|
||||||
|
// CONFIGURATION
|
||||||
|
// host*.conf
|
||||||
|
// <HOST>_SSH_KEY private key this node authenticates to the owner with. Absent or not a
|
||||||
|
// file means no remote route exists, and vv_ai_route() says so rather than
|
||||||
|
// attempting a hop that cannot succeed.
|
||||||
|
// <OWNER> the owner's hostname, looked up by the id vv_ai_owner_host() returns —
|
||||||
|
// resolved to an address through vv_resolve_tailscale_ip(), never used as
|
||||||
|
// a hostname directly, because MagicDNS does not resolve across the tailnets.
|
||||||
|
//
|
||||||
|
// DEPENDS ON
|
||||||
|
// include/config.php vv_ai_owner_host(), vv_ai_is_owner(), vv_resolve_tailscale_ip()
|
||||||
|
// Tools/ai_rpc.php the far side — reached at the WebGUI symlink path
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
require_once __DIR__ . '/config.php';
|
||||||
|
|
||||||
|
const VV_AI_ROUTE_LOCAL = 0;
|
||||||
|
const VV_AI_ROUTE_REMOTE = 1;
|
||||||
|
const VV_AI_ROUTE_DENY = 2;
|
||||||
|
|
||||||
|
// Curated on one node on purpose. Memory is the text that rides in every prompt; a bug report is
|
||||||
|
// what leaves this mesh for a tracker. Both are the owner's to write, and a second node keeping a
|
||||||
|
// divergent copy of either is the failure this refuses.
|
||||||
|
const VV_AI_OWNER_ONLY = ['memory_set', 'mem_proposal_action',
|
||||||
|
'bugs', 'bug_close', 'bug_report', 'bug_send_local'];
|
||||||
|
|
||||||
|
// Answered on the node that asked, even on a mirror. A conversation belongs to the operator in
|
||||||
|
// front of it, and the transcript for a turn travels in the request anyway.
|
||||||
|
const VV_AI_NODE_LOCAL = ['chats', 'chat_get', 'chat_save', 'chat_delete'];
|
||||||
|
|
||||||
|
// Reached over the mesh by a sweep on another node, never by a browser. Denied from the web on
|
||||||
|
// every node — on a mirror because it is not a page action, and on the owner because the only
|
||||||
|
// legitimate caller is the RPC shim, which does not come through here.
|
||||||
|
const VV_AI_RPC_ONLY = ['finding_write'];
|
||||||
|
|
||||||
|
function vv_ai_route(string $action): int {
|
||||||
|
if (in_array($action, VV_AI_RPC_ONLY, true)) return VV_AI_ROUTE_DENY;
|
||||||
|
if (vv_ai_is_owner()) return VV_AI_ROUTE_LOCAL;
|
||||||
|
if (in_array($action, VV_AI_NODE_LOCAL, true)) return VV_AI_ROUTE_LOCAL;
|
||||||
|
if (in_array($action, VV_AI_OWNER_ONLY, true)) return VV_AI_ROUTE_DENY;
|
||||||
|
return VV_AI_ROUTE_REMOTE;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where the multiplexed control socket lives. tmpfs is the right lifetime — a reboot should not
|
||||||
|
// inherit a stale socket — and the path is kept short because a unix socket path is capped near
|
||||||
|
// 108 characters and ssh composes this one with the user and host appended.
|
||||||
|
function vv_ai_rpc_socket_dir(): string {
|
||||||
|
$dir = rtrim(VV_CACHE_ROOT, '/') . '/ssh';
|
||||||
|
if (!is_dir($dir)) @mkdir($dir, 0700, true);
|
||||||
|
return $dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forward one action to the AI owner and return its response body.
|
||||||
|
//
|
||||||
|
// $httpStatus is set from the owner's own status when it reports one, so a 405 raised over there
|
||||||
|
// arrives here as a 405 rather than as a 200 carrying an error string.
|
||||||
|
function vv_ai_rpc(string $action, array $params, bool $isPost, int &$httpStatus = 200): array {
|
||||||
|
$vars = vv_conf_vars();
|
||||||
|
$me = strtoupper(vv_detect_host());
|
||||||
|
$owner = vv_ai_owner_host();
|
||||||
|
|
||||||
|
$sshKey = $vars[$me . '_SSH_KEY'] ?? '';
|
||||||
|
if (!$sshKey || !is_file($sshKey)) {
|
||||||
|
return ['ok' => false, 'error' => "No SSH key for this node ({$me}_SSH_KEY) — cannot reach the AI owner"];
|
||||||
|
}
|
||||||
|
|
||||||
|
$hostname = trim((string)($vars[strtoupper($owner)] ?? ''));
|
||||||
|
if ($hostname === '') {
|
||||||
|
return ['ok' => false, 'error' => "No hostname recorded for the AI owner ($owner)"];
|
||||||
|
}
|
||||||
|
|
||||||
|
$ip = vv_resolve_tailscale_ip($hostname);
|
||||||
|
if (!$ip) {
|
||||||
|
return ['ok' => false, 'error' => "Cannot resolve $hostname on the tailnet — the AI owner is unreachable"];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Which node is asking. Not an authorization claim — the SSH key already settled that — but a
|
||||||
|
// label, so findings and incidents filed from here are stored against the node they describe.
|
||||||
|
$params['_vv_node'] = strtolower(vv_detect_host());
|
||||||
|
|
||||||
|
$remote = '/usr/local/emhttp/plugins/varaverk/Tools/ai_rpc.php';
|
||||||
|
$sock = vv_ai_rpc_socket_dir() . '/ai-%h';
|
||||||
|
|
||||||
|
$cmd = 'ssh -i ' . escapeshellarg($sshKey)
|
||||||
|
. ' -o BatchMode=yes -o StrictHostKeyChecking=no'
|
||||||
|
. ' -o ConnectTimeout=8'
|
||||||
|
. ' -o ControlMaster=auto -o ControlPersist=60s'
|
||||||
|
. ' -o ControlPath=' . escapeshellarg($sock)
|
||||||
|
. ' root@' . escapeshellarg($ip)
|
||||||
|
. ' ' . escapeshellarg('[ -f ' . $remote . ' ] || exit 127; php ' . $remote);
|
||||||
|
|
||||||
|
$desc = [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']];
|
||||||
|
$pr = @proc_open($cmd, $desc, $pipes);
|
||||||
|
if (!is_resource($pr)) {
|
||||||
|
return ['ok' => false, 'error' => 'Cannot start ssh to the AI owner'];
|
||||||
|
}
|
||||||
|
|
||||||
|
fwrite($pipes[0], json_encode([
|
||||||
|
'action' => $action,
|
||||||
|
'params' => $params,
|
||||||
|
'is_post' => $isPost,
|
||||||
|
], JSON_UNESCAPED_SLASHES));
|
||||||
|
fclose($pipes[0]);
|
||||||
|
|
||||||
|
$out = stream_get_contents($pipes[1]); fclose($pipes[1]);
|
||||||
|
$err = stream_get_contents($pipes[2]); fclose($pipes[2]);
|
||||||
|
$rc = proc_close($pr);
|
||||||
|
|
||||||
|
// 127 is the guard above finding no shim — the owner is reachable but has not pulled a build
|
||||||
|
// that has one. Distinguished from a transport failure because the fix is entirely different.
|
||||||
|
if ($rc === 127) {
|
||||||
|
return ['ok' => false, 'error' => 'The AI owner has no Tools/ai_rpc.php — it needs a git pull'];
|
||||||
|
}
|
||||||
|
if ($rc !== 0) {
|
||||||
|
$detail = trim($err) !== '' ? ': ' . mb_substr(trim($err), 0, 200) : '';
|
||||||
|
return ['ok' => false, 'error' => "Cannot reach the AI owner ($hostname)$detail"];
|
||||||
|
}
|
||||||
|
|
||||||
|
$decoded = json_decode(trim($out), true);
|
||||||
|
if (!is_array($decoded)) {
|
||||||
|
return ['ok' => false, 'error' => 'The AI owner returned an unreadable response'];
|
||||||
|
}
|
||||||
|
|
||||||
|
// The shim wraps the body so a status can travel with it. An older owner that answers with a
|
||||||
|
// bare body still works — it simply carries no status, which is the 200 default.
|
||||||
|
if (isset($decoded['_vv_rpc'])) {
|
||||||
|
$httpStatus = (int)($decoded['status'] ?? 200);
|
||||||
|
return is_array($decoded['body'] ?? null) ? $decoded['body'] : ['ok' => false, 'error' => 'Malformed response from the AI owner'];
|
||||||
|
}
|
||||||
|
return $decoded;
|
||||||
|
}
|
||||||
@@ -43,6 +43,31 @@
|
|||||||
//
|
//
|
||||||
// Redacted before it is logged, like every other question in this subsystem.
|
// Redacted before it is logged, like every other question in this subsystem.
|
||||||
//
|
//
|
||||||
|
// EXPORTS
|
||||||
|
// Settings vv_ai_web_enabled(), vv_ai_web_provider(), vv_ai_web_results_max(),
|
||||||
|
// vv_ai_web_timeout(), vv_ai_web_searx_url(), vv_ai_web_degoog_url(),
|
||||||
|
// vv_ai_web_api_key(), vv_ai_web_ready()
|
||||||
|
// — every one reads conf; none of them reaches the network.
|
||||||
|
// Providers vv_ai_web_degoog(), vv_ai_web_searxng(), vv_ai_web_brave(), vv_ai_web_tavily()
|
||||||
|
// — one per backend, each returning the same normalised shape.
|
||||||
|
// Transport vv_ai_web_http(), vv_ai_web_normalise()
|
||||||
|
// Entry vv_ai_web_search(), vv_ai_web_context()
|
||||||
|
// — what the rest of the subsystem calls. Everything above is reachable but only
|
||||||
|
// these two are meant to be: search() picks the provider, context() formats for
|
||||||
|
// the prompt.
|
||||||
|
//
|
||||||
|
// CONFIGURATION
|
||||||
|
// master.conf
|
||||||
|
// AI_WEB_SEARCH_ENABLED master switch; anything but "true" disables the whole subsystem
|
||||||
|
// AI_WEB_SEARCH_PROVIDER which backend vv_ai_web_search() dispatches to
|
||||||
|
// AI_WEB_SEARCH_RESULTS result cap handed to the provider
|
||||||
|
// AI_WEB_SEARCH_TIMEOUT per-request timeout, in seconds
|
||||||
|
//
|
||||||
|
// host*.conf — per host, because the endpoint and the key are this machine's
|
||||||
|
// <HOST>_SEARXNG_URL self-hosted SearXNG base URL
|
||||||
|
// <HOST>_DEGOOG_URL self-hosted degoogle/Whoogle base URL
|
||||||
|
// <HOST>_WEB_SEARCH_API_KEY Brave or Tavily key, whichever provider is selected
|
||||||
|
//
|
||||||
// DEPENDS ON
|
// DEPENDS ON
|
||||||
// include/ai.php vv_ai_redact(), vv_conf_vars()
|
// include/ai.php vv_ai_redact(), vv_conf_vars()
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|||||||
@@ -460,6 +460,12 @@ function vv_arrs_all(): array {
|
|||||||
$cacheFile = VV_CACHE_DIR . '/arrs_remote_' . $h . '.json';
|
$cacheFile = VV_CACHE_DIR . '/arrs_remote_' . $h . '.json';
|
||||||
if (file_exists($cacheFile)) {
|
if (file_exists($cacheFile)) {
|
||||||
$node = json_decode(file_get_contents($cacheFile), true) ?: [];
|
$node = json_decode(file_get_contents($cacheFile), true) ?: [];
|
||||||
|
// The cache file was written by vv_arrs_local_node() ON THAT HOST, where the node is by
|
||||||
|
// definition local — so it arrives claiming 'local' => true and, re-emitted verbatim,
|
||||||
|
// made every node on every host report as this one. Ownership is decided here, by who
|
||||||
|
// is reading the file, not by who wrote it.
|
||||||
|
$node['local'] = false;
|
||||||
|
$node['host'] = $h;
|
||||||
$node['cached'] = true;
|
$node['cached'] = true;
|
||||||
$node['cache_age'] = time() - (int)filemtime($cacheFile);
|
$node['cache_age'] = time() - (int)filemtime($cacheFile);
|
||||||
$result[] = $node;
|
$result[] = $node;
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user