Compare commits
4
Commits
a392108562
...
206a119a4b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
206a119a4b | ||
|
|
671e7ea5a4 | ||
|
|
d5cf3db2ec | ||
|
|
dc8823724d |
Executable
+315
@@ -0,0 +1,315 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
# PURPOSE
|
||||||
|
# Put the containers this host has marked for fallback coverage onto the partner, so that the
|
||||||
|
# partner can actually start them during an outage — and take them off again on request.
|
||||||
|
#
|
||||||
|
# OPERATIONAL MODEL
|
||||||
|
# fallback.sh covers a host by running `docker start <name>` on the partner. It never creates
|
||||||
|
# anything. So a name in FALLBACK_<me>_TIER* is a promise that only holds if the partner already
|
||||||
|
# has that container built. Measured 2026-08-23: all 12 of HOST1's covered containers were absent
|
||||||
|
# from HOST2, meaning every tier would have failed on the first real outage while the UI showed
|
||||||
|
# coverage as configured. This script is what closes that gap.
|
||||||
|
#
|
||||||
|
# Push and remove are separate, deliberate actions, never a side effect of saving the tier list.
|
||||||
|
# Editing coverage is a cheap config write; deploying a dozen containers onto another machine is
|
||||||
|
# not, and the two should not share a button.
|
||||||
|
#
|
||||||
|
# DESIGN PRINCIPLES
|
||||||
|
# Deployed, then verified STOPPED.
|
||||||
|
# A container built here and left running on the partner would be a second live instance of
|
||||||
|
# NextCloud, Gitea or PostgreSQL_Immich against the same data while this host is healthy.
|
||||||
|
# That is the danger_rsync_live_database_appdata failure with worse odds. Every deploy is
|
||||||
|
# followed by a stop and a re-inspect, and a container that will not stay stopped is an
|
||||||
|
# error, not a warning.
|
||||||
|
#
|
||||||
|
# Remove takes the container AND its appdata.
|
||||||
|
# Operator decision 2026-08-23: the button is explicit, so a removal should leave nothing
|
||||||
|
# behind to reason about later. The risk it accepts is narrow and worth naming — if the
|
||||||
|
# partner ever covered for us, ITS appdata is the newer copy and is what a handback rsyncs
|
||||||
|
# home. The NORMAL-state gate below closes the live-failover window; what it cannot see is
|
||||||
|
# a handback that partially failed and then returned to NORMAL, so the UI says so before
|
||||||
|
# asking.
|
||||||
|
#
|
||||||
|
# Two guards on the deletion itself: only paths under /mnt/*/appdata* are ever touched, and
|
||||||
|
# a bind of the appdata ROOT is refused outright — a container mounting /mnt/user/appdata
|
||||||
|
# would otherwise turn one removal into wiping every application on the partner.
|
||||||
|
#
|
||||||
|
# Refuses to run unless fallback state is NORMAL.
|
||||||
|
# Pushing or removing containers mid-outage edits the thing currently keeping services up.
|
||||||
|
#
|
||||||
|
# Coverage names are resolved to templates by <Name>, not by filename.
|
||||||
|
# my-Foo.xml routinely holds a container called something else. Matching on the filename
|
||||||
|
# silently pushes the wrong template, or nothing at all.
|
||||||
|
#
|
||||||
|
# USAGE
|
||||||
|
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
|
||||||
|
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
|
||||||
|
# coverage_deploy.sh --status report, per covered container, whether it exists there
|
||||||
|
# any mode supports --dry-run
|
||||||
|
#
|
||||||
|
# DEPENDS ON
|
||||||
|
# Plugin/<platform>/Partnership/containers.sh deploy_container_from_xml(), GPU transform
|
||||||
|
# FALLBACK_<me>_TIER1-4 the coverage list this acts on
|
||||||
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
||||||
|
|
||||||
|
SSH_TIMEOUT="${SSH_TIMEOUT:-15}"
|
||||||
|
MODE=""
|
||||||
|
DRY_RUN="${DRY_RUN:-false}"
|
||||||
|
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--push) MODE="push" ;;
|
||||||
|
--remove) MODE="remove" ;;
|
||||||
|
--status) MODE="status" ;;
|
||||||
|
--dry-run) DRY_RUN=true ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$MODE" ]]; then
|
||||||
|
error "No mode given — use --push, --remove or --status"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
detect_hosts
|
||||||
|
|
||||||
|
if [[ -z "$REMOTE_ID" || "$REMOTE_SERVER_NAME" == "unknown" ]]; then
|
||||||
|
error "No partner configured — nothing to push to"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Gate: only with fallback idle ─────────────────────────────────────────────────────────────
|
||||||
|
# Read rather than assumed. A missing state file means fallback has never run, which is idle
|
||||||
|
# enough; a file that says anything other than NORMAL means services are in motion right now.
|
||||||
|
FALLBACK_STATE_FILE="${FALLBACK_STATE_FILE:-${STATE_DIR}/fallback_state.db}"
|
||||||
|
_fb_state="NORMAL"
|
||||||
|
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||||
|
_fb_state=$(grep -m1 '^state=' "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||||
|
_fb_state="${_fb_state:-NORMAL}"
|
||||||
|
fi
|
||||||
|
if [[ "$_fb_state" != "NORMAL" && "$MODE" != "status" ]]; then
|
||||||
|
error "Fallback state is $_fb_state, not NORMAL — refusing to $MODE"
|
||||||
|
error "Changing what the partner holds while a failover is live edits the thing keeping services up."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── The coverage list ─────────────────────────────────────────────────────────────────────────
|
||||||
|
COVERED=()
|
||||||
|
for _t in 1 2 3 4; do
|
||||||
|
_var="FALLBACK_${MY_ID}_TIER${_t}[@]"
|
||||||
|
for _c in "${!_var}"; do
|
||||||
|
[[ -n "$_c" ]] && COVERED+=("$_c")
|
||||||
|
done
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${#COVERED[@]} -eq 0 ]]; then
|
||||||
|
warn "No containers are covered in FALLBACK_${MY_ID}_TIER1-4 — nothing to do"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "$ICON_FALLBACK Coverage: ${#COVERED[@]} container(s) for $REMOTE_SERVER_NAME to start during an outage"
|
||||||
|
|
||||||
|
resolve_remote_ip
|
||||||
|
MIRROR="$REMOTE_SERVER_NAME"
|
||||||
|
MIRROR_IP="$REMOTE_SERVER"
|
||||||
|
_key_var="${MY_ID}_SSH_KEY"
|
||||||
|
MIRROR_SSH_KEY="${!_key_var}"
|
||||||
|
|
||||||
|
if [[ ! -f "$MIRROR_SSH_KEY" ]]; then
|
||||||
|
error "SSH key $MIRROR_SSH_KEY not found — cannot reach $MIRROR"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── name -> template ──────────────────────────────────────────────────────────────────────────
|
||||||
|
# Matched on the <Name> element. Filenames lie often enough that trusting them would push the
|
||||||
|
# wrong container without saying so.
|
||||||
|
xml_for_container() {
|
||||||
|
local want="$1" f n
|
||||||
|
for f in "$TEMPLATES_DIR"/*.xml; do
|
||||||
|
[[ -f "$f" ]] || continue
|
||||||
|
n=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$f")
|
||||||
|
[[ "$n" == "$want" ]] && { echo "$f"; return 0; }
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
remote_has_container() {
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker inspect $(printf '%q' "$1") >/dev/null 2>&1" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
remote_state_of() {
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker inspect -f '{{.State.Status}}' $(printf '%q' "$1") 2>/dev/null" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
OK=0; FAIL=0; SKIP=0
|
||||||
|
|
||||||
|
case "$MODE" in
|
||||||
|
|
||||||
|
status)
|
||||||
|
# Written as a cache as well as printed. The assistant's fallback_state block cannot afford an
|
||||||
|
# SSH round trip per container mid-question, so it reads this file and reports its AGE — a stale
|
||||||
|
# answer stated as stale is useful, stated as current it is the exact failure this feature
|
||||||
|
# exists to prevent.
|
||||||
|
_present="" _missing=""
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
if remote_has_container "$c"; then
|
||||||
|
_st=$(remote_state_of "$c")
|
||||||
|
printf ' %-28s on %s (%s)\n' "$c" "$MIRROR" "$_st"
|
||||||
|
_present+="\"$c\":\"${_st:-unknown}\","
|
||||||
|
OK=$((OK+1))
|
||||||
|
else
|
||||||
|
printf ' %-28s MISSING on %s — docker start would fail\n' "$c" "$MIRROR"
|
||||||
|
_missing+="\"$c\","
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "$ICON_FALLBACK Coverage present: $OK · missing: $FAIL"
|
||||||
|
|
||||||
|
mkdir -p "$VV_CACHE_ROOT/api" 2>/dev/null || mkdir -p /tmp/varaverk/api 2>/dev/null
|
||||||
|
_cache="${VV_CACHE_ROOT:-/tmp/varaverk}/api/fallback_presence.json"
|
||||||
|
# Written atomically — a half-written cache read mid-question would report containers as
|
||||||
|
# missing that are merely unparsed.
|
||||||
|
printf '{"present":{%s},"missing":[%s],"partner":"%s","checked":%s}\n' \
|
||||||
|
"${_present%,}" "${_missing%,}" "$MIRROR" "$(date +%s)" > "$_cache.tmp" \
|
||||||
|
&& mv -f "$_cache.tmp" "$_cache"
|
||||||
|
|
||||||
|
[[ "$FAIL" -gt 0 ]] && exit 2 || exit 0
|
||||||
|
;;
|
||||||
|
|
||||||
|
push)
|
||||||
|
# Networks first — a container whose network is absent is created and then cannot start,
|
||||||
|
# which is the failure that read as "auth 0/8, arr 0/5" during onboarding.
|
||||||
|
_nets=()
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
x=$(xml_for_container "$c") || continue
|
||||||
|
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$x" 2>/dev/null | head -1)
|
||||||
|
net="${net//[[:space:]]/}"
|
||||||
|
# br* is host hardware. wg* is a WireGuard-backed bridge whose meaning does NOT travel:
|
||||||
|
# recreating it on the partner as a plain bridge yields a network that exists, starts its
|
||||||
|
# containers, and routes their traffic OUTSIDE the tunnel. ChannelTube rides wg0 here.
|
||||||
|
case "$net" in
|
||||||
|
''|bridge|host|none|br[0-9]*) continue ;;
|
||||||
|
wg[0-9]*)
|
||||||
|
warn "$c uses $net — a WireGuard-backed network. NOT created on $MIRROR: a plain"
|
||||||
|
warn " bridge of the same name would route its traffic outside the tunnel. Build the"
|
||||||
|
warn " matching tunnel there first, or drop $c from coverage."
|
||||||
|
continue ;;
|
||||||
|
esac
|
||||||
|
_seen=false
|
||||||
|
for n in "${_nets[@]}"; do [[ "$n" == "$net" ]] && { _seen=true; break; }; done
|
||||||
|
[[ "$_seen" == false ]] && _nets+=("$net")
|
||||||
|
done
|
||||||
|
for net in "${_nets[@]}"; do
|
||||||
|
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" --format '{{.Driver}}' 2>/dev/null)
|
||||||
|
if [[ "$driver" != "bridge" ]]; then
|
||||||
|
warn "Network $net is '${driver:-absent}' here, not bridge — create it on $MIRROR by hand"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would ensure network $net on $MIRROR"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
||||||
|
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null \
|
||||||
|
&& log " network $net ready on $MIRROR" \
|
||||||
|
|| warn " could not ensure network $net on $MIRROR"
|
||||||
|
done
|
||||||
|
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
x=$(xml_for_container "$c") || {
|
||||||
|
warn "$c — no template in $TEMPLATES_DIR names it; skipped"
|
||||||
|
SKIP=$((SKIP+1)); continue
|
||||||
|
}
|
||||||
|
if ! deploy_container_from_xml "$x" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||||
|
error "$c — deploy failed"
|
||||||
|
FAIL=$((FAIL+1)); continue
|
||||||
|
fi
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then OK=$((OK+1)); continue; fi
|
||||||
|
|
||||||
|
# Deployed containers must not run here. Stop, then re-inspect — a stop that did not take
|
||||||
|
# is the one outcome that silently duplicates a live service against shared data.
|
||||||
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker stop $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null
|
||||||
|
st=$(remote_state_of "$c")
|
||||||
|
if [[ "$st" == "running" ]]; then
|
||||||
|
error "$c is RUNNING on $MIRROR after deploy and would not stop — stop it there before continuing"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
else
|
||||||
|
log " $c deployed and ${st:-stopped} on $MIRROR ✅"
|
||||||
|
OK=$((OK+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "$ICON_FALLBACK Push complete — deployed $OK · failed $FAIL · skipped $SKIP"
|
||||||
|
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||||
|
;;
|
||||||
|
|
||||||
|
remove)
|
||||||
|
for c in "${COVERED[@]}"; do
|
||||||
|
if ! remote_has_container "$c"; then
|
||||||
|
log " $c not on $MIRROR — nothing to remove"
|
||||||
|
SKIP=$((SKIP+1)); continue
|
||||||
|
fi
|
||||||
|
# Binds are read BEFORE the container goes — once it is removed there is nothing left to
|
||||||
|
# enumerate, and a path list gathered afterwards would silently be empty.
|
||||||
|
_binds=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' $(printf '%q' "$c") 2>/dev/null \
|
||||||
|
| awk -F: '{print \$1}'" 2>/dev/null)
|
||||||
|
|
||||||
|
_wipe=()
|
||||||
|
while IFS= read -r _p; do
|
||||||
|
[[ -z "$_p" ]] && continue
|
||||||
|
# Only appdata, and never an appdata root. /mnt/user/appdata as a bind would make one
|
||||||
|
# container removal delete every application on the partner.
|
||||||
|
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/.+ ]] || {
|
||||||
|
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/?$ ]] && \
|
||||||
|
warn " $c binds the appdata ROOT ($_p) — refusing to delete it"
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_wipe+=("$_p")
|
||||||
|
done <<< "$_binds"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would stop and remove $c on $MIRROR"
|
||||||
|
for _p in "${_wipe[@]}"; do warn " DRY RUN — would delete appdata $_p on $MIRROR"; done
|
||||||
|
OK=$((OK+1)); continue
|
||||||
|
fi
|
||||||
|
x=$(xml_for_container "$c") && xml_name=$(basename "$x") || xml_name=""
|
||||||
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"docker stop $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||||
|
docker rm $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||||
|
${xml_name:+rm -f ${TEMPLATES_DIR}/$(printf '%q' "$xml_name");} \
|
||||||
|
! docker inspect $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null; then
|
||||||
|
log " $c removed from $MIRROR ✅"
|
||||||
|
for _p in "${_wipe[@]}"; do
|
||||||
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||||
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||||
|
"rm -rf -- $(printf '%q' "$_p") && ! [ -e $(printf '%q' "$_p") ]" 2>/dev/null; then
|
||||||
|
log " appdata deleted on $MIRROR: $_p"
|
||||||
|
else
|
||||||
|
warn " could not delete appdata on $MIRROR: $_p"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
OK=$((OK+1))
|
||||||
|
else
|
||||||
|
error "$c — removal failed or it still exists on $MIRROR"
|
||||||
|
FAIL=$((FAIL+1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
log "$ICON_FALLBACK Remove complete — removed $OK · failed $FAIL · skipped $SKIP"
|
||||||
|
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||||
|
;;
|
||||||
|
esac
|
||||||
@@ -266,16 +266,8 @@ echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
|||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# The configured list is the denominator — a script the conf names but that never launched is
|
||||||
warn "DRY RUN — no scripts launched"
|
# skipped, not absent, and only shows up if something counts it.
|
||||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
JOB_COUNT="${#ARRAY_START_SCRIPTS[@]}"
|
||||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
orchestrator_summary "ARRAY START" "$START" "Array Start"
|
||||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
exit $?
|
||||||
"Array Start" "warning"
|
|
||||||
else
|
|
||||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
@@ -190,18 +190,6 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
JOB_COUNT="$STEP"
|
||||||
warn "DRY RUN — no changes made"
|
orchestrator_summary "ARRAY STOP" "$START" "Array Stop"
|
||||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
exit $?
|
||||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
|
||||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
|
||||||
"Array Stop" "normal"
|
|
||||||
else
|
|
||||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
|
||||||
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
|
||||||
"Array Stop" "warning"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -280,23 +280,7 @@ fi
|
|||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
DURATION=$(format_duration $(( END - START )))
|
DURATION=$(format_duration $(( END - START )))
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
# Standard ending, quiet mode — 30-min cadence, so a healthy cycle stays one line.
|
||||||
|
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||||
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
orchestrator_summary "CRITICAL SYNC" "$START" "Critical Sync" quiet
|
||||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
exit $?
|
||||||
echo ""
|
|
||||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Duration: $DURATION"
|
|
||||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
|
||||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
|
||||||
"Critical Sync" "warning"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
exit 0
|
|
||||||
@@ -354,11 +354,6 @@ WINDOW_END=$(date +%s)
|
|||||||
# ━━━ Summary ━━━
|
# ━━━ Summary ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||||
for entry in "${SHARE_TIMES[@]}"; do
|
for entry in "${SHARE_TIMES[@]}"; do
|
||||||
@@ -382,16 +377,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
echo ""
|
echo ""
|
||||||
fi
|
fi
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
# Standard ending — derives skipped from SHARE_COUNT, so a run with rsync gated off reports
|
||||||
|
# PARTIAL instead of "all complete".
|
||||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
orchestrator_summary "DAILY MAINTENANCE" "$WINDOW_START" "Daily Maintenance"
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
exit $?
|
||||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
||||||
"Daily Maintenance" "warning"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
exit 1
|
|
||||||
else
|
|
||||||
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
@@ -357,20 +357,8 @@ if [[ "$SHOW_FULL" == true ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
|
||||||
warn "DRY RUN — no changes made"
|
# anything skipped or failed expands to the full block on its own.
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||||
if [[ "$SHOW_FULL" == true ]]; then
|
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
|
||||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
exit $?
|
||||||
else
|
|
||||||
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
|
||||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
||||||
"Intermediate Sync" "warning"
|
|
||||||
fi
|
|
||||||
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -307,18 +307,8 @@ echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped
|
||||||
warn "DRY RUN — no changes made"
|
# step visible rather than absent.
|
||||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
JOB_COUNT="$STEP"
|
||||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance"
|
||||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
exit $?
|
||||||
"Monthly Maintenance" "normal"
|
|
||||||
else
|
|
||||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
|
||||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
|
||||||
"Monthly Maintenance" "warning"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -98,6 +98,9 @@ ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|||||||
|
|
||||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||||
|
|
||||||
|
# Timed from here so the standard summary can report a real duration; this report had none.
|
||||||
|
REPORT_START=$(date +%s)
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -204,10 +207,9 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
# Standard ending. The configured section list is the denominator, so a report that quietly
|
||||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
# stopped producing one of its sections reads as skipped rather than simply not appearing.
|
||||||
"Sunday Morning Coffee Report" "warning"
|
JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}"
|
||||||
fi
|
[[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} ))
|
||||||
|
orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report"
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
exit $?
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -139,6 +139,10 @@ detect_hosts
|
|||||||
|
|
||||||
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
||||||
# from a healthy run. Fail loudly instead of silently doing no work.
|
# from a healthy run. Fail loudly instead of silently doing no work.
|
||||||
|
# This orchestrator never timed itself, so its summary could not report a duration. Set before
|
||||||
|
# any work so the figure means the cycle, not the tail of it.
|
||||||
|
CYCLE_START=$(date +%s)
|
||||||
|
|
||||||
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
if [[ ${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} -eq 0 ]]; then
|
||||||
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
error "TRANSCODE_MANAGEMENT_SCRIPTS is empty — no transcode management scripts will run"
|
||||||
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
error "Check TRANSCODE_MANAGEMENT_SCRIPTS in master.conf"
|
||||||
@@ -223,15 +227,9 @@ done
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "${#JOB_FAIL[@]}" -eq 0 ]]; then
|
# Quiet by default — 7-min cadence. Anything failed or skipped expands on its own.
|
||||||
echo "$ICON_SUCCESS Transcode cycle — ${#JOB_PASS[@]}/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
JOB_COUNT="${#TRANSCODE_MANAGEMENT_SCRIPTS[@]}"
|
||||||
else
|
orchestrator_summary "TRANSCODE CYCLE" "${CYCLE_START:-$(date +%s)}" "Transcode Management" quiet
|
||||||
error "Transcode cycle — failed: ${JOB_FAIL[*]}"
|
|
||||||
if [[ "$DRY_RUN" != true ]]; then
|
|
||||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
|
||||||
"Transcode Management" "warning"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Exit ━━━
|
# ━━━ Exit ━━━
|
||||||
|
|||||||
@@ -241,21 +241,15 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# Per-script detail only when there is something to read; the standard block carries the rest.
|
||||||
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||||
echo ""
|
|
||||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
|
||||||
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
for p in "${JOB_PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||||
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
for f in "${JOB_FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
else
|
|
||||||
echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
|
# Quiet by default at a 15-min cadence. The configured script list is the denominator, so a
|
||||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
# watchdog that silently stopped running one of its checks shows up as skipped.
|
||||||
"Watchdog Orchestrator" "warning"
|
JOB_COUNT="${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||||||
exit 1
|
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
||||||
fi
|
orchestrator_summary "WATCHDOG CYCLE" "$CYCLE_START" "Watchdog Orchestrator" "$_mode"
|
||||||
|
exit $?
|
||||||
exit 0
|
|
||||||
|
|||||||
@@ -402,18 +402,14 @@ WINDOW_END=$(date +%s)
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary ━━━
|
# ━━━ Summary ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# Per-unit detail first — the standard block that follows carries the verdict and the counts, not
|
||||||
|
# the names, and knowing WHICH share failed is the whole point of reading a log.
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
||||||
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):"
|
||||||
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||||
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||||
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
|
||||||
|
|
||||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
@@ -422,19 +418,7 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
# Standard ending. Derives skipped from SHARE_COUNT vs what actually ran, so a gated-off section
|
||||||
|
# can no longer read as success — this is the run that printed "all complete — 0 shares synced".
|
||||||
echo ""
|
orchestrator_summary "WEEKLY SYNC MAINTENANCE" "$WINDOW_START" "Weekly Maintenance"
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
exit $?
|
||||||
warn "DRY RUN — no changes made"
|
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
|
||||||
echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
||||||
else
|
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
|
||||||
notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
||||||
"Weekly Maintenance" "warning"
|
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
||||||
|
|
||||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
||||||
exit 0
|
|
||||||
@@ -174,7 +174,12 @@ transform_xml_for_gpu() {
|
|||||||
local src_xml="$1" gpu_type="$2"
|
local src_xml="$1" gpu_type="$2"
|
||||||
|
|
||||||
# Detect GPU-aware XMLs — new style (--gpus "device=) or old style (--runtime=nvidia)
|
# Detect GPU-aware XMLs — new style (--gpus "device=) or old style (--runtime=nvidia)
|
||||||
if ! grep -qE '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
# -- before the pattern is load-bearing: it begins with "--", so without it grep parses the
|
||||||
|
# pattern as an OPTION, exits 2, and the ! makes this branch always true — the function then
|
||||||
|
# returned the XML untransformed every single time, for every GPU type, with 2>/dev/null
|
||||||
|
# swallowing "invalid option". Every container onboarded to a mirror kept the owner NVIDIA
|
||||||
|
# device UUID and could not start on Intel or AMD hardware. Found 2026-08-23.
|
||||||
|
if ! grep -qE -- '--gpus[[:space:]]+"device=|--runtime=nvidia|NVIDIA_VISIBLE_DEVICES' "$src_xml" 2>/dev/null; then
|
||||||
echo "$src_xml"
|
echo "$src_xml"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -422,6 +422,17 @@ if ($can('system_state')) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fallback is dormant until it isn't, so "configured" and "would work" are unrelated — this is the
|
||||||
|
// only block that reports the second. Cheap: local conf and state file plus one cached presence
|
||||||
|
// read, never a network round trip.
|
||||||
|
if ($can('fallback_state')) {
|
||||||
|
$fb = vv_ai_fallback_state();
|
||||||
|
if ($fb !== '') {
|
||||||
|
$attached['fallback'] = 'readiness';
|
||||||
|
$diagBlock .= $fb;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The troubleshooting profile gets the actual tail of the one log the operator is looking at,
|
// The troubleshooting profile gets the actual tail of the one log the operator is looking at,
|
||||||
// warnings and ordinary lines alike. The fleet-wide WARN/ERROR sweep above cannot answer "why
|
// warnings and ordinary lines alike. The fleet-wide WARN/ERROR sweep above cannot answer "why
|
||||||
// did this one stop" — the last line a script printed before dying is usually not labelled.
|
// did this one stop" — the last line a script printed before dying is usually not labelled.
|
||||||
|
|||||||
@@ -59,6 +59,7 @@
|
|||||||
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf()
|
// include/config.php vv_detect_host(), vv_read_conf_raw(), vv_push_master_conf()
|
||||||
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
// ═══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
header('Content-Type: application/json');
|
header('Content-Type: application/json');
|
||||||
|
require_once dirname(__DIR__) . '/include/fallback.php'; // vv_fb_proc()
|
||||||
require_once dirname(__DIR__) . '/include/confform.php';
|
require_once dirname(__DIR__) . '/include/confform.php';
|
||||||
require_once dirname(__DIR__) . '/include/common.php';
|
require_once dirname(__DIR__) . '/include/common.php';
|
||||||
|
|
||||||
@@ -143,6 +144,191 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Write ────────────────────────────────────────────────────────────────────────────────────
|
// ── Write ────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
// ── Readiness: would a failover actually work right now ─────────────────────────────────────
|
||||||
|
// Every row is a deterministic check with a stated basis. The assistant on this page EXPLAINS
|
||||||
|
// these rows; it never produces them. A model must not be the thing that says failover is ready —
|
||||||
|
// that is precisely the class of answer this codebase keeps finding to be confidently wrong, and
|
||||||
|
// on 2026-08-23 the coverage card itself was the confidently wrong surface: 12 containers listed,
|
||||||
|
// none of them present on the partner.
|
||||||
|
//
|
||||||
|
// Rows are ordered by what breaks first, not by severity, so reading top to bottom follows the
|
||||||
|
// order a real outage would hit them.
|
||||||
|
if (($_POST['action'] ?? '') === 'readiness') {
|
||||||
|
$rows = [];
|
||||||
|
$add = function (string $id, string $label, string $verdict, string $detail, string $ask = '')
|
||||||
|
use (&$rows) {
|
||||||
|
// verdict: ok | warn | fail | unknown — unknown is never dressed up as ok
|
||||||
|
$rows[] = ['id' => $id, 'label' => $label, 'verdict' => $verdict,
|
||||||
|
'detail' => $detail, 'ask' => $ask];
|
||||||
|
};
|
||||||
|
|
||||||
|
$me = vv_detect_host();
|
||||||
|
$ME = strtoupper($me);
|
||||||
|
$conf = vv_read_conf_raw('master.conf');
|
||||||
|
$hc = vv_read_conf_raw($me . '.conf');
|
||||||
|
|
||||||
|
// 1. is fallback even armed
|
||||||
|
$fbEnabled = preg_match('/^\s*FALLBACK_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||||
|
$add('enabled', 'Fallback armed',
|
||||||
|
$fbEnabled === 'true' ? 'ok' : 'fail',
|
||||||
|
'FALLBACK_ENABLED=' . $fbEnabled,
|
||||||
|
'FALLBACK_ENABLED is ' . $fbEnabled . ' — what does that mean for a real outage?');
|
||||||
|
|
||||||
|
// 2. current state — anything but NORMAL means it is already doing something
|
||||||
|
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||||
|
$state = 'unknown';
|
||||||
|
if (is_readable($stateFile) && preg_match('/^state=(\S+)/m', (string)@file_get_contents($stateFile), $m)) {
|
||||||
|
$state = $m[1];
|
||||||
|
}
|
||||||
|
// Same rule the node cards use: no state file plus a live daemon means the node has simply
|
||||||
|
// never transitioned, which is health, not ignorance. Reading the file alone gives a healthy
|
||||||
|
// node the same verdict as one whose daemon is dead.
|
||||||
|
$daemon = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||||
|
$inferred = false;
|
||||||
|
if ($state === 'unknown' && $daemon) { $state = 'NORMAL'; $inferred = true; }
|
||||||
|
|
||||||
|
$add('state', 'State is NORMAL',
|
||||||
|
$state === 'NORMAL' ? 'ok' : ($state === 'unknown' ? 'unknown' : 'warn'),
|
||||||
|
'state=' . $state . ($inferred ? ' (from the live daemon — never transitioned)' : ''),
|
||||||
|
'Fallback state is ' . $state . '. What does that mean and what should I check?');
|
||||||
|
|
||||||
|
// 3. coverage configured at all
|
||||||
|
$covered = [];
|
||||||
|
for ($t = 1; $t <= 4; $t++) {
|
||||||
|
foreach (vv_parse_conf_list($hc, "FALLBACK_{$ME}_TIER{$t}") as $c) $covered[] = $c;
|
||||||
|
}
|
||||||
|
$add('coverage', 'Containers are covered',
|
||||||
|
$covered ? 'ok' : 'fail',
|
||||||
|
$covered ? count($covered) . ' container(s) across the tiers' : 'no containers in any tier',
|
||||||
|
$covered
|
||||||
|
? 'Walk me through what happens if this host goes dark right now, tier by tier, with the delays.'
|
||||||
|
: 'Nothing is listed in my fallback tiers — what would happen if this host went dark?');
|
||||||
|
|
||||||
|
// 4. THE one that was silently false — does the partner actually hold them
|
||||||
|
$cache = (defined('VV_CACHE_ROOT') ? VV_CACHE_ROOT : '/tmp/varaverk') . '/api/fallback_presence.json';
|
||||||
|
if (!is_readable($cache)) {
|
||||||
|
$add('present', 'Partner has the containers', 'unknown',
|
||||||
|
'never checked — run the presence check',
|
||||||
|
'How do I find out whether the partner actually has my covered containers?');
|
||||||
|
} else {
|
||||||
|
$j = json_decode((string)@file_get_contents($cache), true);
|
||||||
|
$miss = (array)($j['missing'] ?? []);
|
||||||
|
$age = time() - (int)@filemtime($cache);
|
||||||
|
$when = $age < 3600 ? round($age / 60) . 'm ago' : round($age / 3600) . 'h ago';
|
||||||
|
$add('present', 'Partner has the containers',
|
||||||
|
$miss ? 'fail' : 'ok',
|
||||||
|
$miss ? count($miss) . ' of ' . count($covered) . ' missing (' . $when . '): '
|
||||||
|
. implode(', ', array_slice($miss, 0, 4)) . (count($miss) > 4 ? '…' : '')
|
||||||
|
: 'all ' . count($covered) . ' present (' . $when . ')',
|
||||||
|
$miss ? 'The partner is missing ' . implode(', ', array_slice($miss, 0, 6))
|
||||||
|
. '. What happens during a failover, and how do I fix it?' : '');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. host-specific networks that cannot be recreated on the partner
|
||||||
|
$wg = [];
|
||||||
|
foreach ($covered as $c) {
|
||||||
|
foreach (glob('/boot/config/plugins/dockerMan/templates-user/*.xml') as $x) {
|
||||||
|
$t = @file_get_contents($x);
|
||||||
|
if ($t === false || strpos($t, "<Name>$c</Name>") === false) continue;
|
||||||
|
if (preg_match('~<Network>(wg\d+)</Network>~', $t, $m)) $wg[] = "$c ({$m[1]})";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($wg) {
|
||||||
|
$add('wgnet', 'No tunnel-bound networks', 'warn',
|
||||||
|
implode(', ', $wg),
|
||||||
|
'Some covered containers use a WireGuard-backed network. Why can that not move to the partner?');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6. handback writeback — invisible until the day it matters
|
||||||
|
$wb = preg_match('/^\s*FALLBACK_RSYNC_ENABLED\s*=\s*"?(\w+)/m', $conf, $m) ? $m[1] : 'unset';
|
||||||
|
$add('writeback', 'Handback writeback', $wb === 'true' ? 'ok' : 'warn',
|
||||||
|
'FALLBACK_RSYNC_ENABLED=' . $wb,
|
||||||
|
'FALLBACK_RSYNC_ENABLED is ' . $wb . ' — what do I lose on handback?');
|
||||||
|
|
||||||
|
// Overall verdict is the worst row, never an average. One failed check is a failed failover.
|
||||||
|
$order = ['ok' => 0, 'warn' => 1, 'unknown' => 2, 'fail' => 3];
|
||||||
|
$worst = 'ok';
|
||||||
|
foreach ($rows as $r) if ($order[$r['verdict']] > $order[$worst]) $worst = $r['verdict'];
|
||||||
|
|
||||||
|
echo json_encode(['ok' => true, 'verdict' => $worst, 'rows' => $rows,
|
||||||
|
'summary' => $worst === 'ok'
|
||||||
|
? 'Every check passed'
|
||||||
|
: ($worst === 'fail' ? 'A failover would NOT work as configured'
|
||||||
|
: 'Failover is configured but something needs a look')]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
// ── Push / remove / status: what the PARTNER actually holds ──────────────────────────────────
|
||||||
|
// Coverage names a container; fallback.sh starts it with `docker start`, which fails unless the
|
||||||
|
// partner already has it built. Measured 2026-08-23: 12 of 12 covered containers were absent from
|
||||||
|
// the partner, so every tier would have failed on the first real outage. These three actions are
|
||||||
|
// how the card closes and inspects that gap.
|
||||||
|
//
|
||||||
|
// Deliberately NOT folded into `cover`. Saving a tier list is a cheap, reversible config write;
|
||||||
|
// deploying a dozen containers onto another machine is neither, and a stray click should not be
|
||||||
|
// able to do it.
|
||||||
|
$_covAction = $_POST['action'] ?? '';
|
||||||
|
if (in_array($_covAction, ['push', 'remove', 'deploy_status'], true)) {
|
||||||
|
$dir = rtrim(SCRIPTS_DIR, '/');
|
||||||
|
$script = $dir . '/Fallback/coverage_deploy.sh';
|
||||||
|
$runner = $dir . '/Plugin/unraid/run_job.sh';
|
||||||
|
|
||||||
|
if (!is_file($script)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'coverage_deploy.sh not found on this host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is read-only and fast enough to answer inline; the two that change the partner are
|
||||||
|
// dispatched to run_job.sh so they get a job record, a log, and a UI surface like every other
|
||||||
|
// long operation here.
|
||||||
|
if ($_covAction === 'deploy_status') {
|
||||||
|
$out = [];
|
||||||
|
exec('timeout 120 /bin/bash ' . escapeshellarg($script) . ' --status 2>&1', $out, $rc);
|
||||||
|
$present = []; $missing = [];
|
||||||
|
foreach ($out as $line) {
|
||||||
|
if (preg_match('/^\s{2}(\S+)\s+MISSING on/', $line, $m)) $missing[] = $m[1];
|
||||||
|
elseif (preg_match('/^\s{2}(\S+)\s+on \S+ \((\w+)\)/', $line, $m)) $present[$m[1]] = $m[2];
|
||||||
|
}
|
||||||
|
echo json_encode([
|
||||||
|
'ok' => true,
|
||||||
|
'present' => $present,
|
||||||
|
'missing' => $missing,
|
||||||
|
// rc 2 means "ran fine, some are missing" — not a failure of the check itself.
|
||||||
|
'checked' => ($rc === 0 || $rc === 2),
|
||||||
|
]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!is_file($runner)) {
|
||||||
|
echo json_encode(['ok' => false, 'error' => 'run_job.sh not found on this host']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
$flag = $_covAction === 'push' ? '--push' : '--remove';
|
||||||
|
$stat = '/var/log/varaverk/Fallback/coverage_deploy.json';
|
||||||
|
|
||||||
|
shell_exec('setsid /bin/bash ' . escapeshellarg($runner)
|
||||||
|
. ' ' . escapeshellarg('Fallback/coverage_deploy.sh')
|
||||||
|
. ' ' . escapeshellarg($script)
|
||||||
|
. ' --manual ' . escapeshellarg($flag)
|
||||||
|
. ' >/dev/null 2>&1 </dev/null &');
|
||||||
|
|
||||||
|
// Report what the record says, not that the command was issued — run_job.sh writes its stat
|
||||||
|
// file before running, so a live record is the difference between a job that started and one
|
||||||
|
// refused for already running, or killed by the NORMAL-state gate.
|
||||||
|
for ($i = 0; $i < 12; $i++) {
|
||||||
|
if (is_file($stat)) {
|
||||||
|
$j = json_decode((string)@file_get_contents($stat), true);
|
||||||
|
if (is_array($j) && ($j['status'] ?? '') === 'running' && time() - filemtime($stat) < 60) {
|
||||||
|
echo json_encode(['ok' => true, 'status' => 'running', 'action' => $_covAction]);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
usleep(250000);
|
||||||
|
}
|
||||||
|
echo json_encode(['ok' => false,
|
||||||
|
'error' => 'Job did not report as running — check the Fallback log. It refuses to run unless fallback state is NORMAL.']);
|
||||||
|
exit;
|
||||||
|
}
|
||||||
if (($_POST['action'] ?? '') !== 'cover') {
|
if (($_POST['action'] ?? '') !== 'cover') {
|
||||||
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
echo json_encode(['ok' => false, 'error' => 'Unknown action']);
|
||||||
exit;
|
exit;
|
||||||
|
|||||||
@@ -1341,8 +1341,11 @@ code.vv-unknown-var { color: #ff9800; background: #1f130d; }
|
|||||||
.vv-df-fname { flex: 1; font-size: 12px; color: #aaa; font-weight: 500;
|
.vv-df-fname { flex: 1; font-size: 12px; color: #aaa; font-weight: 500;
|
||||||
white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
|
||||||
.vv-df-folder-body { padding-left: 10px; padding-bottom: 3px; }
|
.vv-df-folder-body { padding-left: 10px; padding-bottom: 3px; }
|
||||||
.vv-df-container { display: flex; align-items: center; gap: 7px; padding: 3px 6px;
|
/* padding-left is 4px against a 2px border so a container with no fallback tier still lines up
|
||||||
cursor: pointer; border-radius: 3px; user-select: none; }
|
with one that has a stripe — the border is always present, only its colour changes. */
|
||||||
|
.vv-df-container { display: flex; align-items: center; gap: 7px; padding: 3px 6px 3px 4px;
|
||||||
|
cursor: pointer; border-radius: 3px; user-select: none;
|
||||||
|
border-left: 2px solid transparent; }
|
||||||
.vv-df-container:hover { background: rgba(255,255,255,0.04); }
|
.vv-df-container:hover { background: rgba(255,255,255,0.04); }
|
||||||
.vv-df-active { background: rgba(100,149,237,0.1) !important;
|
.vv-df-active { background: rgba(100,149,237,0.1) !important;
|
||||||
outline: 1px solid rgba(100,149,237,0.35);
|
outline: 1px solid rgba(100,149,237,0.35);
|
||||||
|
|||||||
@@ -1483,6 +1483,114 @@ function vv_ai_bug_report(array $b): string {
|
|||||||
// Strictly read-only, and there is no counterpart that changes any of it. Knowing a container is
|
// Strictly read-only, and there is no counterpart that changes any of it. Knowing a container is
|
||||||
// down is what lets an explanation be about this machine instead of about Unraid in general;
|
// down is what lets an explanation be about this machine instead of about Unraid in general;
|
||||||
// restarting it is a decision that belongs to a person looking at the screen.
|
// restarting it is a decision that belongs to a person looking at the screen.
|
||||||
|
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
// Fallback readiness, for the assistant on the Fallback tab.
|
||||||
|
//
|
||||||
|
// Fallback differs from every other subsystem here in one way that shapes this whole function: it
|
||||||
|
// is DORMANT until it isn't. A watchdog leaves strikes and restarts to reason about; fallback
|
||||||
|
// leaves nothing at all until a real outage, so "looks fine" and "would work" are unrelated. On
|
||||||
|
// 2026-08-23 the coverage card showed 12 containers configured and every one of them was absent
|
||||||
|
// from the partner — a failover would have started nothing, and no surface said so.
|
||||||
|
//
|
||||||
|
// So this reports what would ACTUALLY happen, not what is configured to happen, and it is explicit
|
||||||
|
// about the difference between the two.
|
||||||
|
//
|
||||||
|
// Never blocks on the network. Partner presence costs an SSH round trip per container, which is far
|
||||||
|
// too slow for a question already waiting on a model, so it is read from the cache
|
||||||
|
// coverage_deploy.sh --status writes and reported WITH ITS AGE. A stale answer stated as stale is
|
||||||
|
// useful; a stale answer stated as current is the failure this whole feature exists to prevent.
|
||||||
|
// ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
function vv_ai_fallback_state(): string {
|
||||||
|
$me = vv_detect_host();
|
||||||
|
if ($me === '') return '';
|
||||||
|
// vv_detect_host() returns the LOWERCASE slug (host1); the conf keys are uppercase
|
||||||
|
// (FALLBACK_HOST1_TIER1). Building the key from the slug as-is silently matched nothing and
|
||||||
|
// reported "NOTHING is covered" on a host with twelve covered containers — a confidently
|
||||||
|
// wrong answer, which is the one outcome this block must never produce.
|
||||||
|
$ME = strtoupper($me);
|
||||||
|
|
||||||
|
$conf = vv_read_conf_raw('master.conf');
|
||||||
|
if ($conf === '') return '';
|
||||||
|
|
||||||
|
$s = "FALLBACK READINESS (read-only — you cannot change any of it, and you must never tell the "
|
||||||
|
. "operator a failover will work unless the evidence below says so)\n";
|
||||||
|
|
||||||
|
// ── current state ───────────────────────────────────────────────────────────────────────
|
||||||
|
$stateFile = STATE_DIR . '/fallback_state.db';
|
||||||
|
$state = 'unknown'; $since = '';
|
||||||
|
if (is_readable($stateFile)) {
|
||||||
|
$raw = (string) @file_get_contents($stateFile);
|
||||||
|
if (preg_match('/^state=(\S+)/m', $raw, $m)) $state = $m[1];
|
||||||
|
if (preg_match('/^fallback_start=(\d+)/m', $raw, $m) && (int)$m[1] > 0) {
|
||||||
|
$since = ' since ' . date('Y-m-d H:i', (int) $m[1]);
|
||||||
|
}
|
||||||
|
$age = time() - (int) @filemtime($stateFile);
|
||||||
|
// The steady NORMAL path writes nothing, so an old mtime is not staleness — it is quiet.
|
||||||
|
$s .= "- state: $state$since (state file last written "
|
||||||
|
. ($age < 3600 ? round($age / 60) . ' minutes' : round($age / 86400) . ' days') . " ago; "
|
||||||
|
. "the NORMAL path writes nothing, so an old file means nothing has changed)\n";
|
||||||
|
} else {
|
||||||
|
// No file is not the same as not running: fallback.sh writes only on a transition.
|
||||||
|
$live = function_exists('vv_fb_proc') ? (vv_fb_proc('fallback')['running'] ?? false) : false;
|
||||||
|
$s .= $live
|
||||||
|
? "- state: NORMAL (inferred — the daemon is running and has never recorded a transition, "
|
||||||
|
. "so it has written no state file; this is health, not ignorance)\n"
|
||||||
|
: "- state: no state file AND no running daemon — fallback is not operating on this host\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (['FALLBACK_ENABLED', 'FALLBACK_RSYNC_ENABLED'] as $k) {
|
||||||
|
if (preg_match('/^\s*' . $k . '\s*=\s*"?(\w+)"?/m', $conf, $m)) {
|
||||||
|
$s .= "- $k: {$m[1]}"
|
||||||
|
. ($k === 'FALLBACK_RSYNC_ENABLED' && $m[1] !== 'true'
|
||||||
|
? " <- handback writeback is OFF: anything the partner writes while covering "
|
||||||
|
. "for this host never comes home\n" : "\n");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── coverage, tier by tier, with the real delays ─────────────────────────────────────────
|
||||||
|
$hostConf = vv_read_conf_raw($me . '.conf');
|
||||||
|
$covered = [];
|
||||||
|
for ($t = 1; $t <= 4; $t++) {
|
||||||
|
$names = vv_parse_conf_list($hostConf, "FALLBACK_{$ME}_TIER{$t}");
|
||||||
|
if (!$names) continue;
|
||||||
|
$delay = '';
|
||||||
|
if ($t > 1 && preg_match('/^\s*' . $ME . '_TIER' . $t . '_DELAY\s*=\s*"?(\d+)/m', $hostConf, $m)) {
|
||||||
|
$delay = " after {$m[1]} minutes";
|
||||||
|
}
|
||||||
|
$s .= "- tier $t" . ($t === 1 ? ' (immediate)' : $delay) . ': ' . implode(', ', $names) . "\n";
|
||||||
|
foreach ($names as $n) $covered[] = $n;
|
||||||
|
}
|
||||||
|
if (!$covered) {
|
||||||
|
$s .= "- coverage: NOTHING is covered — a failover would start no containers at all\n";
|
||||||
|
return $s . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── does the partner actually have them ─────────────────────────────────────────────────
|
||||||
|
$cache = '/tmp/varaverk/api/fallback_presence.json';
|
||||||
|
if (is_readable($cache)) {
|
||||||
|
$j = json_decode((string) @file_get_contents($cache), true);
|
||||||
|
$age = time() - (int) @filemtime($cache);
|
||||||
|
$miss = (array) ($j['missing'] ?? []);
|
||||||
|
$have = array_keys((array) ($j['present'] ?? []));
|
||||||
|
$when = $age < 3600 ? round($age / 60) . ' minutes ago' : round($age / 3600) . ' hours ago';
|
||||||
|
if ($miss) {
|
||||||
|
$s .= "- ON THE PARTNER (checked $when): " . count($miss) . ' of ' . count($covered)
|
||||||
|
. " covered container(s) DO NOT EXIST there: " . implode(', ', $miss) . "\n"
|
||||||
|
. " fallback.sh starts a covered container with `docker start`; it never creates one, "
|
||||||
|
. "so each of those would fail during a real outage. Push them from the Fallback "
|
||||||
|
. "coverage card.\n";
|
||||||
|
} else {
|
||||||
|
$s .= "- ON THE PARTNER (checked $when): all " . count($have)
|
||||||
|
. " covered container(s) exist there\n";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
$s .= "- ON THE PARTNER: not checked. Say so plainly — whether a failover would actually "
|
||||||
|
. "start anything is UNKNOWN until the coverage card's presence check runs.\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
return $s . "\n";
|
||||||
|
}
|
||||||
|
|
||||||
function vv_ai_system_state(): string {
|
function vv_ai_system_state(): string {
|
||||||
$p = '/tmp/varaverk/api/monitor.json';
|
$p = '/tmp/varaverk/api/monitor.json';
|
||||||
if (!is_readable($p)) return '';
|
if (!is_readable($p)) return '';
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ const VV_AI_PROFILES_DEF = [
|
|||||||
// system_state is read-only and shared with repair. Both need to know a container is down
|
// system_state is read-only and shared with repair. Both need to know a container is down
|
||||||
// or a pool is full to explain anything about this machine rather than about Unraid in
|
// or a pool is full to explain anything about this machine rather than about Unraid in
|
||||||
// general; neither gets a way to act on it, and only repair can change a setting.
|
// general; neither gets a way to act on it, and only repair can change a setting.
|
||||||
'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log',
|
'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log',
|
||||||
'incidents', 'conf_lookup', 'file_bugs'],
|
'incidents', 'conf_lookup', 'file_bugs'],
|
||||||
],
|
],
|
||||||
// The only profile that may change a setting, and the only one not offered as a button.
|
// The only profile that may change a setting, and the only one not offered as a button.
|
||||||
@@ -136,7 +136,7 @@ const VV_AI_PROFILES_DEF = [
|
|||||||
'hint' => 'Works through a finding with you, and can apply a fix you approve.',
|
'hint' => 'Works through a finding with you, and can apply a fix you approve.',
|
||||||
'turns' => 3,
|
'turns' => 3,
|
||||||
'ui' => false,
|
'ui' => false,
|
||||||
'caps' => ['retrieve', 'health', 'system_state', 'run_evidence', 'scoped_log', 'incidents',
|
'caps' => ['retrieve', 'health', 'system_state', 'fallback_state', 'run_evidence', 'scoped_log', 'incidents',
|
||||||
'conf_lookup', 'conf_write', 'probe', 'file_findings', 'phrasebook', 'past_fixes'],
|
'conf_lookup', 'conf_write', 'probe', 'file_findings', 'phrasebook', 'past_fixes'],
|
||||||
],
|
],
|
||||||
];
|
];
|
||||||
@@ -148,6 +148,7 @@ const VV_AI_CAP_MEANING = [
|
|||||||
'kind_filter' => 'the retrieval kind filter the page exposes',
|
'kind_filter' => 'the retrieval kind filter the page exposes',
|
||||||
'health' => 'live health sweep measured at question time — the AI subsystem only',
|
'health' => 'live health sweep measured at question time — the AI subsystem only',
|
||||||
'system_state' => 'read-only view of the machine: hardware, containers, pools, array, UPS',
|
'system_state' => 'read-only view of the machine: hardware, containers, pools, array, UPS',
|
||||||
|
'fallback_state' => 'whether a failover would actually work: state, tiers, and whether the partner really has the covered containers',
|
||||||
'run_evidence' => 'run record and log tail for a script named in the question',
|
'run_evidence' => 'run record and log tail for a script named in the question',
|
||||||
'scoped_log' => 'log tail for whatever the operator currently has open',
|
'scoped_log' => 'log tail for whatever the operator currently has open',
|
||||||
'incidents' => 'operator-written history of what previously went wrong with this thing',
|
'incidents' => 'operator-written history of what previously went wrong with this thing',
|
||||||
|
|||||||
@@ -19,7 +19,8 @@
|
|||||||
//
|
//
|
||||||
// EXPORTS
|
// EXPORTS
|
||||||
// vv_container_webui() WebUI URL for one container, or empty
|
// vv_container_webui() WebUI URL for one container, or empty
|
||||||
// vv_get_docker_folders() folder grouping in the legacy shape
|
// vv_container_tier_map() container name (lowercased) => fallback tier 1-4
|
||||||
|
// vv_get_docker_folders() folder grouping in the legacy shape, each container carrying 'tier'
|
||||||
//
|
//
|
||||||
// CONFIGURATION
|
// CONFIGURATION
|
||||||
// Inherits everything from docker.php — see that file's CONFIGURATION block.
|
// Inherits everything from docker.php — see that file's CONFIGURATION block.
|
||||||
@@ -54,11 +55,31 @@ function vv_container_webui(string $name, array $portMap): string {
|
|||||||
return $url;
|
return $url;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fallback tier for each container, from FALLBACK_<ME>_TIER1..4 in THIS host's own conf —
|
||||||
|
// that list is what the partner starts for us, so a lower tier means it comes back sooner.
|
||||||
|
// Keyed lowercase because conf spelling and docker's spelling of a name need not match.
|
||||||
|
function vv_container_tier_map(): array {
|
||||||
|
$me = strtoupper(vv_detect_host());
|
||||||
|
if ($me === 'UNKNOWN') return [];
|
||||||
|
$raw = vv_read_host_conf_raw(strtolower($me));
|
||||||
|
if ($raw === '') return [];
|
||||||
|
|
||||||
|
$map = [];
|
||||||
|
for ($t = 1; $t <= 4; $t++) {
|
||||||
|
foreach (vv_parse_conf_list($raw, "FALLBACK_{$me}_TIER{$t}") as $name) {
|
||||||
|
$key = strtolower(trim($name));
|
||||||
|
if ($key !== '' && !isset($map[$key])) $map[$key] = $t; // lowest tier wins a duplicate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return $map;
|
||||||
|
}
|
||||||
|
|
||||||
function vv_get_docker_folders(): array {
|
function vv_get_docker_folders(): array {
|
||||||
// Varaverk's own docker_folders.json is the primary store (see include/docker.php) —
|
// Varaverk's own docker_folders.json is the primary store (see include/docker.php) —
|
||||||
// reading folder.view3's mirror directly here left this widget empty on any host
|
// reading folder.view3's mirror directly here left this widget empty on any host
|
||||||
// without that optional third-party plugin installed.
|
// without that optional third-party plugin installed.
|
||||||
$folderData = vv_dk_read_json();
|
$folderData = vv_dk_read_json();
|
||||||
|
$tierMap = vv_container_tier_map();
|
||||||
|
|
||||||
// One docker ps call: names, status, port mappings
|
// One docker ps call: names, status, port mappings
|
||||||
$raw = shell_exec("docker ps -a --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null") ?? '';
|
$raw = shell_exec("docker ps -a --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null") ?? '';
|
||||||
@@ -92,6 +113,7 @@ function vv_get_docker_folders(): array {
|
|||||||
'running' => $running,
|
'running' => $running,
|
||||||
'status' => $status,
|
'status' => $status,
|
||||||
'webui' => vv_container_webui($cname, $portMap),
|
'webui' => vv_container_webui($cname, $portMap),
|
||||||
|
'tier' => $tierMap[strtolower($cname)] ?? null,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
usort($containers, fn($a, $b) => $b['running'] <=> $a['running'] ?: strcmp($a['name'], $b['name']));
|
usort($containers, fn($a, $b) => $b['running'] <=> $a['running'] ?: strcmp($a['name'], $b['name']));
|
||||||
@@ -114,6 +136,7 @@ function vv_get_docker_folders(): array {
|
|||||||
'running' => $running,
|
'running' => $running,
|
||||||
'status' => $status,
|
'status' => $status,
|
||||||
'webui' => vv_container_webui($cname, $portMap),
|
'webui' => vv_container_webui($cname, $portMap),
|
||||||
|
'tier' => $tierMap[strtolower($cname)] ?? null,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
usort($ungrouped, fn($a, $b) => strcmp($a['name'], $b['name']));
|
usort($ungrouped, fn($a, $b) => strcmp($a['name'], $b['name']));
|
||||||
|
|||||||
@@ -73,6 +73,8 @@ function vv_fb_parse_state(string $text): array {
|
|||||||
'tier4_started' => false,
|
'tier4_started' => false,
|
||||||
'partnership_suspended' => false,
|
'partnership_suspended' => false,
|
||||||
'partner_lost_at' => 0,
|
'partner_lost_at' => 0,
|
||||||
|
// Set only when the verdict came from the daemon rather than the file — see vv_fb_all().
|
||||||
|
'inferred' => false,
|
||||||
];
|
];
|
||||||
foreach (explode("\n", $text) as $line) {
|
foreach (explode("\n", $text) as $line) {
|
||||||
$line = trim($line);
|
$line = trim($line);
|
||||||
@@ -313,6 +315,29 @@ function vv_fb_all(): array {
|
|||||||
: vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']);
|
: vv_fb_remote_dryrun_state($ip, $mySshKey, (int)$proc['pid']);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── A live daemon that has simply never transitioned ────────────────────────────────
|
||||||
|
// fallback.sh writes its state file ONLY on a transition; the steady NORMAL path writes
|
||||||
|
// nothing at all. So a node that has run cleanly since it was built has no file, and
|
||||||
|
// reading the file alone reports it as UNKNOWN — the same verdict given to a node whose
|
||||||
|
// daemon is dead. Those are opposite conditions and they were rendered identically.
|
||||||
|
//
|
||||||
|
// Observed on HOST2 2026-08-23: daemon live (pid 2208657, valid lock), tailscale and ssh
|
||||||
|
// both fine, no state file, card said UNKNOWN.
|
||||||
|
//
|
||||||
|
// The rule this page is built on — never claim healthy for a host you cannot verify — is
|
||||||
|
// kept: this host CAN be verified, just not from the file that was being consulted. The
|
||||||
|
// daemon holding a live lock is the evidence. reach['state_file'] still reports false,
|
||||||
|
// because there genuinely is no file; 'inferred' says where the verdict came from instead.
|
||||||
|
// Captured before the inference below rewrites it: reach[state_file] must keep answering
|
||||||
|
// "was there a file", not "do we have a verdict". Deriving it after inference made the
|
||||||
|
// card claim a state file existed on a host that has none.
|
||||||
|
$hadStateFile = ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN';
|
||||||
|
|
||||||
|
if (($state['state'] ?? 'UNKNOWN') === 'UNKNOWN' && ($proc['running'] ?? null) === true) {
|
||||||
|
$state['state'] = 'NORMAL';
|
||||||
|
$state['inferred'] = true;
|
||||||
|
}
|
||||||
|
|
||||||
// How fresh the state actually is. The daemon rewrites its file every check interval, so
|
// How fresh the state actually is. The daemon rewrites its file every check interval, so
|
||||||
// an age far past that interval means it is wedged even while the process still exists.
|
// an age far past that interval means it is wedged even while the process still exists.
|
||||||
$stateAge = null;
|
$stateAge = null;
|
||||||
@@ -326,7 +351,7 @@ function vv_fb_all(): array {
|
|||||||
'tailscale' => $isMe ? true : ($ts['online'] === true),
|
'tailscale' => $isMe ? true : ($ts['online'] === true),
|
||||||
'ip' => $isMe ? null : $ip,
|
'ip' => $isMe ? null : $ip,
|
||||||
'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)),
|
'ssh' => $isMe ? true : ($running !== [] || ($proc['running'] !== null)),
|
||||||
'state_file' => ($state['state'] ?? 'UNKNOWN') !== 'UNKNOWN',
|
'state_file' => $hadStateFile,
|
||||||
];
|
];
|
||||||
|
|
||||||
$nodes[] = [
|
$nodes[] = [
|
||||||
|
|||||||
@@ -64,6 +64,28 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
|||||||
border-radius:4px;padding:6px 9px;margin-bottom:8px; }
|
border-radius:4px;padding:6px 9px;margin-bottom:8px; }
|
||||||
.vv-fb-tierhdr-t { font-size:12px;font-weight:700;color:#ffb74d;letter-spacing:.01em; }
|
.vv-fb-tierhdr-t { font-size:12px;font-weight:700;color:#ffb74d;letter-spacing:.01em; }
|
||||||
.vv-fb-tierhdr-s { font-size:10px;color:#7a6038; }
|
.vv-fb-tierhdr-s { font-size:10px;color:#7a6038; }
|
||||||
|
/* ── Failover readiness ──────────────────────────────────────────────────────────────────────
|
||||||
|
Every other card on this page shows what is CONFIGURED. This one shows what would actually
|
||||||
|
happen, which on 2026-08-23 turned out to be a different thing entirely — twelve containers
|
||||||
|
configured, none of them present on the partner, and no surface said so.
|
||||||
|
Verdict colour always ships beside a word, never alone. */
|
||||||
|
.vv-fb-rd { display:flex;flex-direction:column;gap:4px; }
|
||||||
|
.vv-fb-rdrow { display:flex;align-items:center;gap:9px;padding:5px 8px;border-radius:3px;
|
||||||
|
background:#0d0d0d;border:1px solid #161616;border-left:3px solid var(--rv,#333); }
|
||||||
|
.vv-fb-rdrow.ok { --rv:#4caf50; }
|
||||||
|
.vv-fb-rdrow.warn { --rv:#ffb74d; }
|
||||||
|
.vv-fb-rdrow.fail { --rv:#ef5350; }
|
||||||
|
.vv-fb-rdrow.unknown { --rv:#5a7a8a; }
|
||||||
|
.vv-fb-rdv { font-size:9px;font-weight:700;letter-spacing:.05em;text-transform:uppercase;
|
||||||
|
min-width:56px;color:var(--rv,#666); }
|
||||||
|
.vv-fb-rdl { font-size:12px;color:#b8b8b8;min-width:190px; }
|
||||||
|
.vv-fb-rdd { font-size:11px;color:#5a5a5a;flex:1;min-width:0;overflow:hidden;
|
||||||
|
text-overflow:ellipsis;white-space:nowrap; }
|
||||||
|
.vv-fb-rdwhy { font-size:10px;padding:2px 8px;border-radius:3px;cursor:pointer;
|
||||||
|
background:#0e1a2a;color:#7ab;border:1px solid #1e3a5a;white-space:nowrap; }
|
||||||
|
.vv-fb-rdwhy:hover { background:#12233a; }
|
||||||
|
.vv-fb-rdsum { font-size:11px;font-weight:600;margin-bottom:7px; }
|
||||||
|
|
||||||
/* ── Fallback coverage ── */
|
/* ── Fallback coverage ── */
|
||||||
/* One continuum, worst outcome to best: never comes back → 24h → 12h → 4h → immediate → never
|
/* One continuum, worst outcome to best: never comes back → 24h → 12h → 4h → immediate → never
|
||||||
goes down at all. The colour answers "how long am I without this if the partner takes over",
|
goes down at all. The colour answers "how long am I without this if the partner takes over",
|
||||||
@@ -267,6 +289,18 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
|||||||
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
<div style="grid-column:1/-1;color:#444;font-size:12px;padding:16px 0;text-align:center;">Loading…</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Failover readiness — deterministic checks. The assistant EXPLAINS these rows and never
|
||||||
|
produces them: a model must not be the thing that says a failover will work. -->
|
||||||
|
<div class="vv-card" id="vv-fb-readiness" style="margin-bottom:12px;">
|
||||||
|
<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;">
|
||||||
|
<h3 style="margin:0;">Failover readiness</h3>
|
||||||
|
<span style="font-size:10px;color:#444;">would a failover actually work right now</span>
|
||||||
|
<span style="flex:1;"></span>
|
||||||
|
<button class="vv-fb-save-btn" onclick="vvFbReadiness(true)" title="Re-run the checks">↻</button>
|
||||||
|
</div>
|
||||||
|
<div class="vv-fb-rdsum" id="vv-fb-rdsum">checking…</div>
|
||||||
|
<div class="vv-fb-rd" id="vv-fb-rdrows"></div>
|
||||||
|
</div>
|
||||||
<!-- Fallback coverage — this host's own tiers -->
|
<!-- Fallback coverage — this host's own tiers -->
|
||||||
<!--
|
<!--
|
||||||
Originally on the Partnership page (e8ee5b0), removed the same day in 1a836da because it
|
Originally on the Partnership page (e8ee5b0), removed the same day in 1a836da because it
|
||||||
@@ -289,7 +323,23 @@ if (vv_ai_ui_on()) vv_ai_chat_assets();
|
|||||||
To change what <span id="vv-fb-cov-partner" style="color:#666;">the partner</span> hands to us, open this page there.
|
To change what <span id="vv-fb-cov-partner" style="color:#666;">the partner</span> hands to us, open this page there.
|
||||||
</div>
|
</div>
|
||||||
<div id="vv-fb-cov-body" style="color:#444;font-size:12px;">Loading…</div>
|
<div id="vv-fb-cov-body" style="color:#444;font-size:12px;">Loading…</div>
|
||||||
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-top:10px;padding-top:8px;border-top:1px solid #1e1e1e;">
|
<!-- Deploy row, deliberately separate from Save. Save writes the tier list; these two change
|
||||||
|
what the partner physically holds. fallback.sh starts a covered container with
|
||||||
|
`docker start`, which fails unless it was built there first — so a coverage list the
|
||||||
|
partner has never been sent is a promise nothing can keep, and this row is where that
|
||||||
|
is made visible and fixed. -->
|
||||||
|
<div id="vv-fb-cov-deploy" style="display:flex;align-items:center;gap:8px;flex-wrap:wrap;margin-top:10px;padding-top:8px;border-top:1px solid #1e1e1e;">
|
||||||
|
<span style="font-size:10px;color:#5a5a5a;text-transform:uppercase;letter-spacing:.04em;">On partner</span>
|
||||||
|
<span id="vv-fb-cov-presence" style="font-size:11px;color:#444;">checking…</span>
|
||||||
|
<span style="flex:1;"></span>
|
||||||
|
<span id="vv-fb-cov-dfb" style="font-size:11px;"></span>
|
||||||
|
<button class="vv-fb-save-btn" id="vv-fb-cov-push" onclick="vvFbCovDeploy('push')"
|
||||||
|
title="Build every covered container on the partner, left stopped, so a failover can start them">Push to partner</button>
|
||||||
|
<button class="vv-fb-save-btn" id="vv-fb-cov-rm" onclick="vvFbCovDeploy('remove')"
|
||||||
|
style="background:#1a1208;color:#c88;border-color:#3a2a1a;"
|
||||||
|
title="Stop and remove these containers on the partner, and delete their appdata there. Not reversible.">Remove from partner</button>
|
||||||
|
</div>
|
||||||
|
<div style="display:flex;justify-content:flex-end;align-items:center;gap:10px;margin-top:8px;">
|
||||||
<span id="vv-fb-cov-fb" style="font-size:11px;"></span>
|
<span id="vv-fb-cov-fb" style="font-size:11px;"></span>
|
||||||
<button class="vv-fb-save-btn" id="vv-fb-cov-save" onclick="vvFbCovSave()">Save coverage</button>
|
<button class="vv-fb-save-btn" id="vv-fb-cov-save" onclick="vvFbCovSave()">Save coverage</button>
|
||||||
</div>
|
</div>
|
||||||
@@ -654,13 +704,14 @@ function _nodeCard(node, data) {
|
|||||||
${node.is_me ? '<span class="vv-fb-usbadge">US</span>' : ''}
|
${node.is_me ? '<span class="vv-fb-usbadge">US</span>' : ''}
|
||||||
<span style="flex:1"></span>
|
<span style="flex:1"></span>
|
||||||
${(node.is_me && node.proc && node.proc.running && node.proc.mode === 'live') ? _ptStatus(st) : ''}
|
${(node.is_me && node.proc && node.proc.running && node.proc.mode === 'live') ? _ptStatus(st) : ''}
|
||||||
${_stateBadge(shown)}${dryRun ? '<span class="vv-fb-leg" style="margin-left:4px;">preview</span>' : ''}
|
${_stateBadge(shown)}${dryRun ? '<span class="vv-fb-leg" style="margin-left:4px;">preview</span>' : ''}${(st.inferred && !dryRun) ? '<span class="vv-fb-leg" style="margin-left:4px;" title="Inferred from the running daemon rather than read from a state file — this node has never transitioned.">inferred</span>' : ''}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="vv-fb-legs">
|
<div class="vv-fb-legs">
|
||||||
${_leg(reach.tailscale, 'tailscale')}
|
${_leg(reach.tailscale, 'tailscale')}
|
||||||
${_leg(node.is_me ? true : reach.ssh, 'ssh')}
|
${_leg(node.is_me ? true : reach.ssh, 'ssh')}
|
||||||
${_leg(reach.state_file, 'state file')}
|
${_leg(reach.state_file, 'state file')}
|
||||||
|
${st.inferred ? '<span class="vv-fb-leg" title="fallback.sh writes its state file only on a transition, so a node that has run cleanly since it was built has none. The verdict comes from the live daemon holding a valid lock.">· never transitioned</span>' : ''}
|
||||||
${node.ts_ip ? `<span class="vv-fb-leg">${vvEscHtml(node.ts_ip)}</span>` : ''}
|
${node.ts_ip ? `<span class="vv-fb-leg">${vvEscHtml(node.ts_ip)}</span>` : ''}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -1081,9 +1132,174 @@ window.vvFbCovSave = async function () {
|
|||||||
fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e;
|
fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e;
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function _vvFbPartnerName() {
|
||||||
|
const el = document.getElementById('vv-fb-cov-partner');
|
||||||
|
const t = el ? el.textContent.trim() : '';
|
||||||
|
return (t && t !== 'the partner') ? t : 'the partner';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Deploy: what the partner actually holds ───────────────────────────────────────────────────
|
||||||
|
// Presence is read from the partner, never inferred from the tier list. The whole point of this
|
||||||
|
// row is that the two disagree — coverage said 12 containers, the partner had none of them.
|
||||||
|
window.vvFbCovPresence = async function () {
|
||||||
|
const el = document.getElementById('vv-fb-cov-presence');
|
||||||
|
if (!el) return;
|
||||||
|
try {
|
||||||
|
const fd = new URLSearchParams({ action: 'deploy_status' });
|
||||||
|
const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php',
|
||||||
|
{ method: 'POST', body: fd })).json();
|
||||||
|
if (!d.ok || !d.checked) { el.style.color = '#a05a2c'; el.textContent = d.error || 'could not check'; return; }
|
||||||
|
const have = Object.keys(d.present || {}).length, miss = (d.missing || []).length;
|
||||||
|
if (miss === 0 && have === 0) { el.style.color = '#444'; el.textContent = 'nothing covered'; }
|
||||||
|
else if (miss === 0) { el.style.color = '#4caf50'; el.textContent = `all ${have} present`; }
|
||||||
|
else {
|
||||||
|
el.style.color = '#ef5350';
|
||||||
|
// Named, not just counted: "3 missing" is a number, the names are what you act on.
|
||||||
|
el.textContent = `${miss} missing — ${(d.missing||[]).slice(0,3).join(', ')}${miss>3?` +${miss-3}`:''}`;
|
||||||
|
}
|
||||||
|
} catch (e) { el.style.color = '#a05a2c'; el.textContent = 'check failed'; }
|
||||||
|
};
|
||||||
|
|
||||||
|
window.vvFbCovDeploy = async function (which) {
|
||||||
|
const push = which === 'push';
|
||||||
|
const fbEl = document.getElementById('vv-fb-cov-dfb');
|
||||||
|
const btn = document.getElementById(push ? 'vv-fb-cov-push' : 'vv-fb-cov-rm');
|
||||||
|
const n = _vvFbCov ? Object.keys(_vvFbCov.cover).length : 0;
|
||||||
|
|
||||||
|
const msg = push
|
||||||
|
? `Build ${n} container${n!==1?'s':''} on ${_vvFbPartnerName()}?\n\n`
|
||||||
|
+ 'Each is created and left STOPPED so a failover can start it. Nothing starts running now.\n\n'
|
||||||
|
+ 'Save first if you have unsaved changes — this pushes what is in the conf, not what is on screen.'
|
||||||
|
: `Stop and remove ${n} container${n!==1?'s':''} on ${_vvFbPartnerName()} AND DELETE THEIR APPDATA?\n\n`
|
||||||
|
+ 'Not reversible. Only paths under /mnt/*/appdata* are touched; a bind of the appdata root is refused.\n\n'
|
||||||
|
+ 'If the partner has ever covered for this host, what it holds may be the NEWER copy — the one a '
|
||||||
|
+ 'handback rsyncs home. Fallback state is NORMAL, so nothing is failing over right now, but a '
|
||||||
|
+ 'handback that partly failed would not show up here.\n\n'
|
||||||
|
+ 'Coverage stays as configured, so a later Push rebuilds the containers from scratch.';
|
||||||
|
if (!await vvConfirm(msg, { title: push ? 'Push to partner' : 'Remove from partner',
|
||||||
|
confirmText: push ? 'Push' : 'Remove' })) return;
|
||||||
|
|
||||||
|
btn.disabled = true; const label = btn.textContent; btn.textContent = push ? 'Pushing…' : 'Removing…';
|
||||||
|
fbEl.style.color = '#7ab'; fbEl.textContent = 'job started…';
|
||||||
|
try {
|
||||||
|
const fd = new URLSearchParams({ action: which });
|
||||||
|
const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php',
|
||||||
|
{ method: 'POST', body: fd })).json();
|
||||||
|
if (!d.ok) { fbEl.style.color = '#ef5350'; fbEl.textContent = d.error || 'Failed'; }
|
||||||
|
else {
|
||||||
|
// The job runs past this response. Re-checking presence is the only honest completion
|
||||||
|
// signal available here, so poll it rather than claiming success on dispatch.
|
||||||
|
fbEl.style.color = '#7ab'; fbEl.textContent = 'running — see the Fallback log';
|
||||||
|
let ticks = 0;
|
||||||
|
const t = setInterval(async () => {
|
||||||
|
await vvFbCovPresence();
|
||||||
|
// Presence just changed, so the readiness verdict that depends on it is stale.
|
||||||
|
vvFbReadiness(false);
|
||||||
|
if (++ticks >= 20) { clearInterval(t); fbEl.textContent = ''; }
|
||||||
|
}, 6000);
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
fbEl.style.color = '#ef5350'; fbEl.textContent = 'Request failed: ' + e;
|
||||||
|
}
|
||||||
|
btn.disabled = false; btn.textContent = label;
|
||||||
|
};
|
||||||
|
// ── Assistant ────────────────────────────────────────────────────────────────────────────────
|
||||||
|
// vv_ai_chat_markup() above emits the boxes and nothing else — no <script>, no init. Without this
|
||||||
|
// block the card renders looking complete and dies on the first click with VvAiChat undefined.
|
||||||
|
// Fallback was the only one of seven pages mounting a dock and never instantiating it, which is
|
||||||
|
// exactly the failure watchdog.php warns about in its own comment.
|
||||||
|
//
|
||||||
|
// troubleshoot, not varaverk: the placeholder invites "what would the partner start if this host
|
||||||
|
// went dark", and a docs-only profile cannot reach live state to answer it. On 2026-08-23 the
|
||||||
|
// documented answer would also have been wrong — coverage listed 12 containers and none of them
|
||||||
|
// existed on the partner.
|
||||||
|
let vvFbChat = null;
|
||||||
|
let vvFbScope = 'Fallback';
|
||||||
|
|
||||||
|
if (typeof VvAiChat === 'function' && document.getElementById('vv-fb-ai-chat')) {
|
||||||
|
vvFbChat = VvAiChat({
|
||||||
|
prefix: 'vv-fb-ai',
|
||||||
|
profile: 'troubleshoot',
|
||||||
|
scopeLabel: 'Fallback',
|
||||||
|
// Read at send time rather than captured — a Why? retargets the scope and sends from the
|
||||||
|
// same click.
|
||||||
|
scope: () => vvFbScope,
|
||||||
|
// Pinned for the same reason Monitor pins its own: without it the card resumes whatever
|
||||||
|
// thread was last touched anywhere, landing this tab mid-conversation under a profile it
|
||||||
|
// never offers.
|
||||||
|
resumeProfile: 'troubleshoot',
|
||||||
|
think: p => p === 'troubleshoot',
|
||||||
|
empty: 'Ask about fallback — why a tier has not fired, what the partner would actually '
|
||||||
|
+ 'start if this host went dark, whether a stale state file matters.',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retarget the assistant at one thing on the page, then ask about it — same shape as the Watchdog
|
||||||
|
// Why? buttons, and deliberately using the component's real API. There is no ask(): it is
|
||||||
|
// retarget() + set the input + send(), and calling a method that does not exist would fail
|
||||||
|
// silently on click, which is the bug this page already had once.
|
||||||
|
window.vvFbWhy = function (label, question, scope) {
|
||||||
|
if (!vvFbChat || vvFbChat.busy()) return;
|
||||||
|
// troubleshoot, never repair. This page arms and disarms failover; repair is the one profile
|
||||||
|
// that can write conf, and a chat box is the wrong place to do that from.
|
||||||
|
vvFbScope = scope || label;
|
||||||
|
vvFbChat.retarget('troubleshoot', label, 'now looking at ' + label);
|
||||||
|
const input = document.getElementById('vv-fb-ai-input');
|
||||||
|
if (input) input.value = question;
|
||||||
|
vvFbChat.send();
|
||||||
|
const card = document.getElementById('vv-fb-ai-card');
|
||||||
|
if (card) card.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||||
|
};
|
||||||
|
|
||||||
|
// ── Failover readiness ───────────────────────────────────────────────────────────────────────
|
||||||
|
// Rendered from the endpoint's verdicts verbatim. Nothing here decides anything — if a row says
|
||||||
|
// fail, it is because a check failed, not because the page inferred it.
|
||||||
|
window.vvFbReadiness = async function (force) {
|
||||||
|
const sum = document.getElementById('vv-fb-rdsum');
|
||||||
|
const rows = document.getElementById('vv-fb-rdrows');
|
||||||
|
if (!sum || !rows) return;
|
||||||
|
if (force) { sum.textContent = 'checking…'; sum.style.color = '#7ab'; }
|
||||||
|
|
||||||
|
try {
|
||||||
|
const d = await (await fetch('/plugins/varaverk/api/fallback_coverage.php',
|
||||||
|
{ method: 'POST', body: new URLSearchParams({ action: 'readiness' }) })).json();
|
||||||
|
if (!d.ok) throw new Error(d.error || 'no verdict');
|
||||||
|
|
||||||
|
const tone = { ok: '#4caf50', warn: '#ffb74d', fail: '#ef5350', unknown: '#5a7a8a' };
|
||||||
|
sum.style.color = tone[d.verdict] || '#888';
|
||||||
|
sum.textContent = d.summary;
|
||||||
|
|
||||||
|
rows.innerHTML = (d.rows || []).map(r =>
|
||||||
|
`<div class="vv-fb-rdrow ${vvEscAttr(r.verdict)}">`
|
||||||
|
+ `<span class="vv-fb-rdv">${vvEscHtml(r.verdict)}</span>`
|
||||||
|
+ `<span class="vv-fb-rdl">${vvEscHtml(r.label)}</span>`
|
||||||
|
+ `<span class="vv-fb-rdd" title="${vvEscAttr(r.detail)}">${vvEscHtml(r.detail)}</span>`
|
||||||
|
+ (r.ask ? `<span class="vv-fb-rdwhy" data-ask="${vvEscAttr(r.ask)}" `
|
||||||
|
+ `data-label="${vvEscAttr(r.label)}">Why?</span>` : '')
|
||||||
|
+ `</div>`).join('');
|
||||||
|
} catch (e) {
|
||||||
|
sum.style.color = '#ef5350';
|
||||||
|
sum.textContent = 'Could not run the checks — ' + e;
|
||||||
|
rows.innerHTML = '';
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Delegated: the rows are rebuilt on every refresh, so per-node handlers would leak. Neither the
|
||||||
|
// question nor the label is interpolated into an onclick — vvEscHtml does not escape quotes.
|
||||||
|
(function () {
|
||||||
|
const host = document.getElementById('vv-fb-rdrows');
|
||||||
|
if (host) host.addEventListener('click', ev => {
|
||||||
|
const b = ev.target.closest('.vv-fb-rdwhy');
|
||||||
|
if (b) vvFbWhy(b.dataset.label || 'Fallback readiness', b.dataset.ask || 'What does this mean?');
|
||||||
|
});
|
||||||
|
})();
|
||||||
|
|
||||||
vvFbLoad();
|
vvFbLoad();
|
||||||
setInterval(vvFbLoad, 30000);
|
setInterval(vvFbLoad, 30000);
|
||||||
vvFbCovLoad(); // once — this is an editor, not a monitor; polling would fight the operator
|
vvFbCovLoad(); // once — this is an editor, not a monitor; polling would fight the operator
|
||||||
|
vvFbCovPresence();
|
||||||
|
vvFbReadiness(false);
|
||||||
|
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -2975,6 +2975,12 @@ function vvRenderDockerFolders(data) {
|
|||||||
const stateColor = s => ({ running:'#4caf50', paused:'#ff9800' })[s] ?? '#444';
|
const stateColor = s => ({ running:'#4caf50', paused:'#ff9800' })[s] ?? '#444';
|
||||||
const stateLabel = s => ({ running:'Running', paused:'Paused', 'shut off':'Off', crashed:'Crashed' })[s] ?? s;
|
const stateLabel = s => ({ running:'Running', paused:'Paused', 'shut off':'Off', crashed:'Crashed' })[s] ?? s;
|
||||||
|
|
||||||
|
// Fallback tier stripe. Warm (comes back immediately) to cool (waits 24h), riding the row's
|
||||||
|
// left border rather than the status dot — the dot already means running vs stopped, and
|
||||||
|
// overloading it would make a stopped tier-1 container indistinguishable from a running one.
|
||||||
|
const tierColor = t => ({ 1:'#ff5252', 2:'#ffa726', 3:'#ffd54f', 4:'#4fc3f7' })[t] ?? '';
|
||||||
|
const tierWhen = t => ({ 1:'immediately', 2:'after 4h', 3:'after 12h', 4:'after 24h' })[t] ?? '';
|
||||||
|
|
||||||
// ── VMs section ─────────────────────────────────────────────────────────────
|
// ── VMs section ─────────────────────────────────────────────────────────────
|
||||||
let html = '';
|
let html = '';
|
||||||
const vms = data.vms?.vms ?? [];
|
const vms = data.vms?.vms ?? [];
|
||||||
@@ -3033,7 +3039,14 @@ function vvRenderDockerFolders(data) {
|
|||||||
class="vv-btn-sm vv-edit-btn">✎ Edit</button>
|
class="vv-btn-sm vv-edit-btn">✎ Edit</button>
|
||||||
</div>`;
|
</div>`;
|
||||||
}
|
}
|
||||||
return `<div class="vv-df-container${active ? ' vv-df-active' : ''}"
|
// Tier arrives as an int from conf, but it is still payload — anything outside 1-4 gets no
|
||||||
|
// stripe rather than being interpolated into the style attribute.
|
||||||
|
const tNum = Number(c.tier);
|
||||||
|
const tOk = Number.isInteger(tNum) && tNum >= 1 && tNum <= 4;
|
||||||
|
const tSty = tOk ? ` style="border-left-color:${tierColor(tNum)};"` : '';
|
||||||
|
const tTip = tOk ? ` title="Fallback tier ${tNum} — partner starts this ${tierWhen(tNum)}"` : '';
|
||||||
|
|
||||||
|
return `<div class="vv-df-container${active ? ' vv-df-active' : ''}"${tSty}${tTip}
|
||||||
onclick="event.stopPropagation();vvToggleContainer('${sn}')">
|
onclick="event.stopPropagation();vvToggleContainer('${sn}')">
|
||||||
<span class="vv-df-dot" style="background:${dot};${pulse}"></span>
|
<span class="vv-df-dot" style="background:${dot};${pulse}"></span>
|
||||||
<span class="vv-df-cname">${vvEscHtml(c.name)}</span>
|
<span class="vv-df-cname">${vvEscHtml(c.name)}</span>
|
||||||
|
|||||||
@@ -3507,4 +3507,96 @@ require_partnership() {
|
|||||||
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
[[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0
|
||||||
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
log "PARTNERSHIP_ENABLED=false — skipping cross-server operation"
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
# ── Standardised orchestrator ending ─────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# Every orchestrator closes with this, so a run's verdict reads the same everywhere and can be
|
||||||
|
# parsed by one rule instead of ten.
|
||||||
|
#
|
||||||
|
# WHY IT EXISTS: on 2026-08-23 the Sunday weekly ran 2h43m, exited 0, and printed
|
||||||
|
# "Status: all complete ✅ — 0 share(s) synced"
|
||||||
|
# because its two sync jobs were SKIPPED, not failed — TOTAL_FAIL was 0, so it declared success.
|
||||||
|
# Nothing in the old per-script summaries modelled work that was expected and never attempted, so
|
||||||
|
# a gated-off section was indistinguishable from a clean run. Skipped is a first-class outcome here.
|
||||||
|
#
|
||||||
|
# VERDICTS — the headline degrades, never flatters:
|
||||||
|
# DRY RUN nothing was changed
|
||||||
|
# FAILED one or more units failed
|
||||||
|
# PARTIAL nothing failed, but expected work was skipped <- the case that used to read OK
|
||||||
|
# IDLE there was genuinely nothing to do
|
||||||
|
# OK every expected unit ran and passed
|
||||||
|
#
|
||||||
|
# READS these globals if set, treating absent as empty — the names every orchestrator already uses:
|
||||||
|
# PASS FAIL per-share outcomes SHARE_COUNT shares expected
|
||||||
|
# JOB_PASS JOB_FAIL per-job outcomes JOB_COUNT jobs expected (optional)
|
||||||
|
# DRY_RUN MY_ID LOCAL_SERVER_NAME
|
||||||
|
#
|
||||||
|
# ARGUMENTS
|
||||||
|
# $1 display name for the run, e.g. "WEEKLY SYNC MAINTENANCE"
|
||||||
|
# $2 start epoch
|
||||||
|
# $3 optional: notification subject; omitted means do not notify
|
||||||
|
#
|
||||||
|
# RETURNS 0 for DRY RUN / IDLE / OK / PARTIAL, 1 for FAILED — so `exit $?` is the whole contract.
|
||||||
|
# PARTIAL returns 0 deliberately: skipped work is usually a toggle the operator set on purpose, and
|
||||||
|
# a non-zero exit would make cron mail every gated run. It is loud in the log, not in the exit code.
|
||||||
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||||
|
# $4 "quiet" — for high-cadence runs (7/15 min). Prints ONLY the RESULT line when the verdict
|
||||||
|
# is OK or IDLE, and the full block otherwise. A cycle that skipped or failed something is
|
||||||
|
# never quiet, which is the only case anyone greps for anyway.
|
||||||
|
orchestrator_summary() {
|
||||||
|
local title="$1" start_ts="$2" subject="${3:-}" mode="${4:-full}"
|
||||||
|
local end_ts; end_ts=$(date +%s)
|
||||||
|
|
||||||
|
local sp=${#PASS[@]} sf=${#FAIL[@]}
|
||||||
|
local jp=${#JOB_PASS[@]} jf=${#JOB_FAIL[@]}
|
||||||
|
local sc="${SHARE_COUNT:-$(( sp + sf ))}"
|
||||||
|
local jc="${JOB_COUNT:-$(( jp + jf ))}"
|
||||||
|
|
||||||
|
# Skipped is derived, never reported by the caller — a section that bails early cannot be
|
||||||
|
# relied on to remember to say so, which is exactly how this was missed for weeks.
|
||||||
|
local ss=$(( sc - sp - sf )); [[ "$ss" -lt 0 ]] && ss=0
|
||||||
|
local js=$(( jc - jp - jf )); [[ "$js" -lt 0 ]] && js=0
|
||||||
|
|
||||||
|
local total=$(( sc + jc )) fails=$(( sf + jf )) skips=$(( ss + js ))
|
||||||
|
local verdict icon
|
||||||
|
if [[ "${DRY_RUN:-false}" == true ]]; then verdict="DRY RUN"; icon="$ICON_WARN"
|
||||||
|
elif [[ "$fails" -gt 0 ]]; then verdict="FAILED"; icon="$ICON_ERROR"
|
||||||
|
elif [[ "$skips" -gt 0 ]]; then verdict="PARTIAL"; icon="$ICON_WARN"
|
||||||
|
elif [[ "$total" -eq 0 ]]; then verdict="IDLE"; icon="$ICON_INFO"
|
||||||
|
else verdict="OK"; icon="$ICON_DONE"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Quiet cycles collapse to the one parseable line — but only when there is nothing to see.
|
||||||
|
if [[ "$mode" == quiet && ( "$verdict" == OK || "$verdict" == IDLE ) ]]; then
|
||||||
|
echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY $title — $verdict ━━━━━"
|
||||||
|
echo "$ICON_HOST Identity: ${MY_ID:-unknown} (${LOCAL_SERVER_NAME:-$(hostname)})"
|
||||||
|
echo "$ICON_TIME Window: $(date -d @"$start_ts" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$end_ts" '+%H:%M:%S')"
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $(( end_ts - start_ts )))"
|
||||||
|
[[ "$sc" -gt 0 ]] && echo "$ICON_SYNC Shares: $sp ok · $sf failed · $ss skipped (of $sc)"
|
||||||
|
[[ "$jc" -gt 0 ]] && echo "$ICON_GEAR Jobs: $jp ok · $jf failed · $js skipped (of $jc)"
|
||||||
|
|
||||||
|
[[ "$sf" -gt 0 ]] && { echo "$ICON_ERROR Failed shares: ${FAIL[*]}"; }
|
||||||
|
[[ "$jf" -gt 0 ]] && { echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"; }
|
||||||
|
|
||||||
|
# One machine-readable line, always last and always the same shape, so the board and any log
|
||||||
|
# scraper have a single thing to match instead of ten prose variants.
|
||||||
|
echo "$icon RESULT verdict=$verdict shares=$sp/$sf/$ss jobs=$jp/$jf/$js expected=$total duration=$(( end_ts - start_ts ))s"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
if [[ -n "$subject" && "${DRY_RUN:-false}" != true ]]; then
|
||||||
|
if [[ "$verdict" == "FAILED" ]]; then
|
||||||
|
notify "$title FAILED on $(hostname) (${MY_ID:-?}) — $fails of $total unit(s) failed" "$subject" "warning"
|
||||||
|
elif [[ "$verdict" == "PARTIAL" ]]; then
|
||||||
|
notify "$title PARTIAL on $(hostname) (${MY_ID:-?}) — $skips of $total unit(s) skipped, none failed" "$subject" "warning"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$verdict" == "FAILED" ]] && return 1
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user