Make fallback coverage something that can actually happen, and say so on the page
fallback.sh starts covered containers with docker start and never creates them, so a coverage list the partner has never been sent is a promise nothing can keep — all twelve were missing. Adds the push and remove paths, a readiness card that checks rather than infers, and the fallback state the assistant needs to answer for it.
This commit is contained in:
Gmer4Lfe
Executable
+315
@@ -0,0 +1,315 @@
|
||||
#!/bin/bash
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
# PURPOSE
|
||||
# Put the containers this host has marked for fallback coverage onto the partner, so that the
|
||||
# partner can actually start them during an outage — and take them off again on request.
|
||||
#
|
||||
# OPERATIONAL MODEL
|
||||
# fallback.sh covers a host by running `docker start <name>` on the partner. It never creates
|
||||
# anything. So a name in FALLBACK_<me>_TIER* is a promise that only holds if the partner already
|
||||
# has that container built. Measured 2026-08-23: all 12 of HOST1's covered containers were absent
|
||||
# from HOST2, meaning every tier would have failed on the first real outage while the UI showed
|
||||
# coverage as configured. This script is what closes that gap.
|
||||
#
|
||||
# Push and remove are separate, deliberate actions, never a side effect of saving the tier list.
|
||||
# Editing coverage is a cheap config write; deploying a dozen containers onto another machine is
|
||||
# not, and the two should not share a button.
|
||||
#
|
||||
# DESIGN PRINCIPLES
|
||||
# Deployed, then verified STOPPED.
|
||||
# A container built here and left running on the partner would be a second live instance of
|
||||
# NextCloud, Gitea or PostgreSQL_Immich against the same data while this host is healthy.
|
||||
# That is the danger_rsync_live_database_appdata failure with worse odds. Every deploy is
|
||||
# followed by a stop and a re-inspect, and a container that will not stay stopped is an
|
||||
# error, not a warning.
|
||||
#
|
||||
# Remove takes the container AND its appdata.
|
||||
# Operator decision 2026-08-23: the button is explicit, so a removal should leave nothing
|
||||
# behind to reason about later. The risk it accepts is narrow and worth naming — if the
|
||||
# partner ever covered for us, ITS appdata is the newer copy and is what a handback rsyncs
|
||||
# home. The NORMAL-state gate below closes the live-failover window; what it cannot see is
|
||||
# a handback that partially failed and then returned to NORMAL, so the UI says so before
|
||||
# asking.
|
||||
#
|
||||
# Two guards on the deletion itself: only paths under /mnt/*/appdata* are ever touched, and
|
||||
# a bind of the appdata ROOT is refused outright — a container mounting /mnt/user/appdata
|
||||
# would otherwise turn one removal into wiping every application on the partner.
|
||||
#
|
||||
# Refuses to run unless fallback state is NORMAL.
|
||||
# Pushing or removing containers mid-outage edits the thing currently keeping services up.
|
||||
#
|
||||
# Coverage names are resolved to templates by <Name>, not by filename.
|
||||
# my-Foo.xml routinely holds a container called something else. Matching on the filename
|
||||
# silently pushes the wrong template, or nothing at all.
|
||||
#
|
||||
# USAGE
|
||||
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
|
||||
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
|
||||
# coverage_deploy.sh --status report, per covered container, whether it exists there
|
||||
# any mode supports --dry-run
|
||||
#
|
||||
# DEPENDS ON
|
||||
# Plugin/<platform>/Partnership/containers.sh deploy_container_from_xml(), GPU transform
|
||||
# FALLBACK_<me>_TIER1-4 the coverage list this acts on
|
||||
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
||||
|
||||
SSH_TIMEOUT="${SSH_TIMEOUT:-15}"
|
||||
MODE=""
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--push) MODE="push" ;;
|
||||
--remove) MODE="remove" ;;
|
||||
--status) MODE="status" ;;
|
||||
--dry-run) DRY_RUN=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$MODE" ]]; then
|
||||
error "No mode given — use --push, --remove or --status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
if [[ -z "$REMOTE_ID" || "$REMOTE_SERVER_NAME" == "unknown" ]]; then
|
||||
error "No partner configured — nothing to push to"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Gate: only with fallback idle ─────────────────────────────────────────────────────────────
|
||||
# Read rather than assumed. A missing state file means fallback has never run, which is idle
|
||||
# enough; a file that says anything other than NORMAL means services are in motion right now.
|
||||
FALLBACK_STATE_FILE="${FALLBACK_STATE_FILE:-${STATE_DIR}/fallback_state.db}"
|
||||
_fb_state="NORMAL"
|
||||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||||
_fb_state=$(grep -m1 '^state=' "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
_fb_state="${_fb_state:-NORMAL}"
|
||||
fi
|
||||
if [[ "$_fb_state" != "NORMAL" && "$MODE" != "status" ]]; then
|
||||
error "Fallback state is $_fb_state, not NORMAL — refusing to $MODE"
|
||||
error "Changing what the partner holds while a failover is live edits the thing keeping services up."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── The coverage list ─────────────────────────────────────────────────────────────────────────
|
||||
COVERED=()
|
||||
for _t in 1 2 3 4; do
|
||||
_var="FALLBACK_${MY_ID}_TIER${_t}[@]"
|
||||
for _c in "${!_var}"; do
|
||||
[[ -n "$_c" ]] && COVERED+=("$_c")
|
||||
done
|
||||
done
|
||||
|
||||
if [[ ${#COVERED[@]} -eq 0 ]]; then
|
||||
warn "No containers are covered in FALLBACK_${MY_ID}_TIER1-4 — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "$ICON_FALLBACK Coverage: ${#COVERED[@]} container(s) for $REMOTE_SERVER_NAME to start during an outage"
|
||||
|
||||
resolve_remote_ip
|
||||
MIRROR="$REMOTE_SERVER_NAME"
|
||||
MIRROR_IP="$REMOTE_SERVER"
|
||||
_key_var="${MY_ID}_SSH_KEY"
|
||||
MIRROR_SSH_KEY="${!_key_var}"
|
||||
|
||||
if [[ ! -f "$MIRROR_SSH_KEY" ]]; then
|
||||
error "SSH key $MIRROR_SSH_KEY not found — cannot reach $MIRROR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── name -> template ──────────────────────────────────────────────────────────────────────────
|
||||
# Matched on the <Name> element. Filenames lie often enough that trusting them would push the
|
||||
# wrong container without saying so.
|
||||
xml_for_container() {
|
||||
local want="$1" f n
|
||||
for f in "$TEMPLATES_DIR"/*.xml; do
|
||||
[[ -f "$f" ]] || continue
|
||||
n=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$f")
|
||||
[[ "$n" == "$want" ]] && { echo "$f"; return 0; }
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
remote_has_container() {
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker inspect $(printf '%q' "$1") >/dev/null 2>&1" 2>/dev/null
|
||||
}
|
||||
|
||||
remote_state_of() {
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker inspect -f '{{.State.Status}}' $(printf '%q' "$1") 2>/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
OK=0; FAIL=0; SKIP=0
|
||||
|
||||
case "$MODE" in
|
||||
|
||||
status)
|
||||
# Written as a cache as well as printed. The assistant's fallback_state block cannot afford an
|
||||
# SSH round trip per container mid-question, so it reads this file and reports its AGE — a stale
|
||||
# answer stated as stale is useful, stated as current it is the exact failure this feature
|
||||
# exists to prevent.
|
||||
_present="" _missing=""
|
||||
for c in "${COVERED[@]}"; do
|
||||
if remote_has_container "$c"; then
|
||||
_st=$(remote_state_of "$c")
|
||||
printf ' %-28s on %s (%s)\n' "$c" "$MIRROR" "$_st"
|
||||
_present+="\"$c\":\"${_st:-unknown}\","
|
||||
OK=$((OK+1))
|
||||
else
|
||||
printf ' %-28s MISSING on %s — docker start would fail\n' "$c" "$MIRROR"
|
||||
_missing+="\"$c\","
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
done
|
||||
log "$ICON_FALLBACK Coverage present: $OK · missing: $FAIL"
|
||||
|
||||
mkdir -p "$VV_CACHE_ROOT/api" 2>/dev/null || mkdir -p /tmp/varaverk/api 2>/dev/null
|
||||
_cache="${VV_CACHE_ROOT:-/tmp/varaverk}/api/fallback_presence.json"
|
||||
# Written atomically — a half-written cache read mid-question would report containers as
|
||||
# missing that are merely unparsed.
|
||||
printf '{"present":{%s},"missing":[%s],"partner":"%s","checked":%s}\n' \
|
||||
"${_present%,}" "${_missing%,}" "$MIRROR" "$(date +%s)" > "$_cache.tmp" \
|
||||
&& mv -f "$_cache.tmp" "$_cache"
|
||||
|
||||
[[ "$FAIL" -gt 0 ]] && exit 2 || exit 0
|
||||
;;
|
||||
|
||||
push)
|
||||
# Networks first — a container whose network is absent is created and then cannot start,
|
||||
# which is the failure that read as "auth 0/8, arr 0/5" during onboarding.
|
||||
_nets=()
|
||||
for c in "${COVERED[@]}"; do
|
||||
x=$(xml_for_container "$c") || continue
|
||||
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$x" 2>/dev/null | head -1)
|
||||
net="${net//[[:space:]]/}"
|
||||
# br* is host hardware. wg* is a WireGuard-backed bridge whose meaning does NOT travel:
|
||||
# recreating it on the partner as a plain bridge yields a network that exists, starts its
|
||||
# containers, and routes their traffic OUTSIDE the tunnel. ChannelTube rides wg0 here.
|
||||
case "$net" in
|
||||
''|bridge|host|none|br[0-9]*) continue ;;
|
||||
wg[0-9]*)
|
||||
warn "$c uses $net — a WireGuard-backed network. NOT created on $MIRROR: a plain"
|
||||
warn " bridge of the same name would route its traffic outside the tunnel. Build the"
|
||||
warn " matching tunnel there first, or drop $c from coverage."
|
||||
continue ;;
|
||||
esac
|
||||
_seen=false
|
||||
for n in "${_nets[@]}"; do [[ "$n" == "$net" ]] && { _seen=true; break; }; done
|
||||
[[ "$_seen" == false ]] && _nets+=("$net")
|
||||
done
|
||||
for net in "${_nets[@]}"; do
|
||||
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" --format '{{.Driver}}' 2>/dev/null)
|
||||
if [[ "$driver" != "bridge" ]]; then
|
||||
warn "Network $net is '${driver:-absent}' here, not bridge — create it on $MIRROR by hand"
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would ensure network $net on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
||||
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null \
|
||||
&& log " network $net ready on $MIRROR" \
|
||||
|| warn " could not ensure network $net on $MIRROR"
|
||||
done
|
||||
|
||||
for c in "${COVERED[@]}"; do
|
||||
x=$(xml_for_container "$c") || {
|
||||
warn "$c — no template in $TEMPLATES_DIR names it; skipped"
|
||||
SKIP=$((SKIP+1)); continue
|
||||
}
|
||||
if ! deploy_container_from_xml "$x" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
||||
error "$c — deploy failed"
|
||||
FAIL=$((FAIL+1)); continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then OK=$((OK+1)); continue; fi
|
||||
|
||||
# Deployed containers must not run here. Stop, then re-inspect — a stop that did not take
|
||||
# is the one outcome that silently duplicates a live service against shared data.
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker stop $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null
|
||||
st=$(remote_state_of "$c")
|
||||
if [[ "$st" == "running" ]]; then
|
||||
error "$c is RUNNING on $MIRROR after deploy and would not stop — stop it there before continuing"
|
||||
FAIL=$((FAIL+1))
|
||||
else
|
||||
log " $c deployed and ${st:-stopped} on $MIRROR ✅"
|
||||
OK=$((OK+1))
|
||||
fi
|
||||
done
|
||||
log "$ICON_FALLBACK Push complete — deployed $OK · failed $FAIL · skipped $SKIP"
|
||||
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||
;;
|
||||
|
||||
remove)
|
||||
for c in "${COVERED[@]}"; do
|
||||
if ! remote_has_container "$c"; then
|
||||
log " $c not on $MIRROR — nothing to remove"
|
||||
SKIP=$((SKIP+1)); continue
|
||||
fi
|
||||
# Binds are read BEFORE the container goes — once it is removed there is nothing left to
|
||||
# enumerate, and a path list gathered afterwards would silently be empty.
|
||||
_binds=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' $(printf '%q' "$c") 2>/dev/null \
|
||||
| awk -F: '{print \$1}'" 2>/dev/null)
|
||||
|
||||
_wipe=()
|
||||
while IFS= read -r _p; do
|
||||
[[ -z "$_p" ]] && continue
|
||||
# Only appdata, and never an appdata root. /mnt/user/appdata as a bind would make one
|
||||
# container removal delete every application on the partner.
|
||||
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/.+ ]] || {
|
||||
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/?$ ]] && \
|
||||
warn " $c binds the appdata ROOT ($_p) — refusing to delete it"
|
||||
continue
|
||||
}
|
||||
_wipe+=("$_p")
|
||||
done <<< "$_binds"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would stop and remove $c on $MIRROR"
|
||||
for _p in "${_wipe[@]}"; do warn " DRY RUN — would delete appdata $_p on $MIRROR"; done
|
||||
OK=$((OK+1)); continue
|
||||
fi
|
||||
x=$(xml_for_container "$c") && xml_name=$(basename "$x") || xml_name=""
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"docker stop $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||
docker rm $(printf '%q' "$c") >/dev/null 2>&1; \
|
||||
${xml_name:+rm -f ${TEMPLATES_DIR}/$(printf '%q' "$xml_name");} \
|
||||
! docker inspect $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null; then
|
||||
log " $c removed from $MIRROR ✅"
|
||||
for _p in "${_wipe[@]}"; do
|
||||
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
||||
"rm -rf -- $(printf '%q' "$_p") && ! [ -e $(printf '%q' "$_p") ]" 2>/dev/null; then
|
||||
log " appdata deleted on $MIRROR: $_p"
|
||||
else
|
||||
warn " could not delete appdata on $MIRROR: $_p"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
done
|
||||
OK=$((OK+1))
|
||||
else
|
||||
error "$c — removal failed or it still exists on $MIRROR"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
done
|
||||
log "$ICON_FALLBACK Remove complete — removed $OK · failed $FAIL · skipped $SKIP"
|
||||
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
||||
;;
|
||||
esac
|
||||
Reference in New Issue
Block a user