352 lines
17 KiB
Bash
Executable File
352 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
|
# PURPOSE
|
|
# Put the containers this host has marked for fallback coverage onto the partner, so that the
|
|
# partner can actually start them during an outage — and take them off again on request.
|
|
#
|
|
# OPERATIONAL MODEL
|
|
# fallback.sh covers a host by running `docker start <name>` on the partner. It never creates
|
|
# anything. So a name in FALLBACK_<me>_TIER* is a promise that only holds if the partner already
|
|
# has that container built. Measured 2026-08-23: all 12 of HOST1's covered containers were absent
|
|
# from HOST2, meaning every tier would have failed on the first real outage while the UI showed
|
|
# coverage as configured. This script is what closes that gap.
|
|
#
|
|
# Push and remove are separate, deliberate actions, never a side effect of saving the tier list.
|
|
# Editing coverage is a cheap config write; deploying a dozen containers onto another machine is
|
|
# not, and the two should not share a button.
|
|
#
|
|
# DESIGN PRINCIPLES
|
|
# Deployed, then verified STOPPED.
|
|
# A container built here and left running on the partner would be a second live instance of
|
|
# NextCloud, Gitea or PostgreSQL_Immich against the same data while this host is healthy.
|
|
# That is the danger_rsync_live_database_appdata failure with worse odds. Every deploy is
|
|
# followed by a stop and a re-inspect, and a container that will not stay stopped is an
|
|
# error, not a warning.
|
|
#
|
|
# Remove takes the container AND its appdata.
|
|
# Operator decision 2026-08-23: the button is explicit, so a removal should leave nothing
|
|
# behind to reason about later. The risk it accepts is narrow and worth naming — if the
|
|
# partner ever covered for us, ITS appdata is the newer copy and is what a handback rsyncs
|
|
# home. The NORMAL-state gate below closes the live-failover window; what it cannot see is
|
|
# a handback that partially failed and then returned to NORMAL, so the UI says so before
|
|
# asking.
|
|
#
|
|
# Two guards on the deletion itself: only paths under /mnt/*/appdata* are ever touched, and
|
|
# a bind of the appdata ROOT is refused outright — a container mounting /mnt/user/appdata
|
|
# would otherwise turn one removal into wiping every application on the partner.
|
|
#
|
|
# Refuses to run unless fallback state is NORMAL.
|
|
# Pushing or removing containers mid-outage edits the thing currently keeping services up.
|
|
#
|
|
# Coverage names are resolved to templates by <Name>, not by filename.
|
|
# my-Foo.xml routinely holds a container called something else. Matching on the filename
|
|
# silently pushes the wrong template, or nothing at all.
|
|
#
|
|
# OPERATIONAL SAFEGUARDS
|
|
# Only in NORMAL state. FALLBACK_STATE_FILE is read before anything is pushed or removed, and
|
|
# any other state refuses the action. A push during a live failover would deploy a second copy
|
|
# of a container the partner is currently running on our behalf; a remove would delete the one
|
|
# doing the covering.
|
|
#
|
|
# --status is exempt from that gate, because it only reports. Refusing to answer "what is
|
|
# deployed over there" during a failover would withhold the information precisely when it is
|
|
# most wanted.
|
|
#
|
|
# Every deploy is verified stopped, and a container that will not stay stopped is an error
|
|
# rather than a warning — see DESIGN PRINCIPLES. A second live instance against the same data
|
|
# is the failure this whole script exists inside.
|
|
#
|
|
# Push and remove are explicit modes with no default. Running the script with no flag does
|
|
# nothing; neither action can be reached by accident, and neither is a side effect of editing
|
|
# the tier list.
|
|
#
|
|
# --dry-run works in every mode and touches nothing on either host — no container is built,
|
|
# started, stopped or removed, and no template is written or deleted.
|
|
#
|
|
# Remove deletes the container's appdata on the partner as well. That is deliberate and is the
|
|
# most destructive thing here; the NORMAL-state gate above is what keeps it away from a
|
|
# partner that is mid-handback.
|
|
#
|
|
# CONFIGURATION
|
|
# master.conf
|
|
# FALLBACK_<HOST>_TIER1..N the covered container names — what --push deploys and --status
|
|
# reports on. This script reads that list; it never edits it.
|
|
#
|
|
# host*.conf
|
|
# FALLBACK_STATE_FILE overrides where fallback.sh's state is read from. Defaults to
|
|
# STATE_DIR/fallback_state.db. A missing file reads as NORMAL,
|
|
# which is the correct default on a host where fallback has never
|
|
# run.
|
|
#
|
|
# RUNTIME MODES
|
|
# coverage_deploy.sh --push deploy every covered container onto the partner (stopped)
|
|
# coverage_deploy.sh --remove stop, remove, and delete the pushed template on the partner
|
|
# coverage_deploy.sh --status report, per covered container, whether it exists there
|
|
# any mode supports --dry-run
|
|
#
|
|
# DEPENDS ON
|
|
# Plugin/<platform>/Partnership/containers.sh deploy_container_from_xml(), GPU transform
|
|
# FALLBACK_<me>_TIER1-4 the coverage list this acts on
|
|
# ══════════════════════════════════════════════════════════════════════════════════════════════
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
source "$SCRIPT_DIR/../Plugin/$PLATFORM/Partnership/containers.sh"
|
|
|
|
SSH_TIMEOUT="${SSH_TIMEOUT:-15}"
|
|
MODE=""
|
|
DRY_RUN="${DRY_RUN:-false}"
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--push) MODE="push" ;;
|
|
--remove) MODE="remove" ;;
|
|
--status) MODE="status" ;;
|
|
--dry-run) DRY_RUN=true ;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$MODE" ]]; then
|
|
error "No mode given — use --push, --remove or --status"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
if [[ -z "$REMOTE_ID" || "$REMOTE_SERVER_NAME" == "unknown" ]]; then
|
|
error "No partner configured — nothing to push to"
|
|
exit 1
|
|
fi
|
|
|
|
# ── Gate: only with fallback idle ─────────────────────────────────────────────────────────────
|
|
# Read rather than assumed. A missing state file means fallback has never run, which is idle
|
|
# enough; a file that says anything other than NORMAL means services are in motion right now.
|
|
FALLBACK_STATE_FILE="${FALLBACK_STATE_FILE:-${STATE_DIR}/fallback_state.db}"
|
|
_fb_state="NORMAL"
|
|
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
|
_fb_state=$(grep -m1 '^state=' "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
|
_fb_state="${_fb_state:-NORMAL}"
|
|
fi
|
|
if [[ "$_fb_state" != "NORMAL" && "$MODE" != "status" ]]; then
|
|
error "Fallback state is $_fb_state, not NORMAL — refusing to $MODE"
|
|
error "Changing what the partner holds while a failover is live edits the thing keeping services up."
|
|
exit 1
|
|
fi
|
|
|
|
# ── The coverage list ─────────────────────────────────────────────────────────────────────────
|
|
COVERED=()
|
|
for _t in 1 2 3 4; do
|
|
_var="FALLBACK_${MY_ID}_TIER${_t}[@]"
|
|
for _c in "${!_var}"; do
|
|
[[ -n "$_c" ]] && COVERED+=("$_c")
|
|
done
|
|
done
|
|
|
|
if [[ ${#COVERED[@]} -eq 0 ]]; then
|
|
warn "No containers are covered in FALLBACK_${MY_ID}_TIER1-4 — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
log "$ICON_FALLBACK Coverage: ${#COVERED[@]} container(s) for $REMOTE_SERVER_NAME to start during an outage"
|
|
|
|
resolve_remote_ip
|
|
MIRROR="$REMOTE_SERVER_NAME"
|
|
MIRROR_IP="$REMOTE_SERVER"
|
|
_key_var="${MY_ID}_SSH_KEY"
|
|
MIRROR_SSH_KEY="${!_key_var}"
|
|
|
|
if [[ ! -f "$MIRROR_SSH_KEY" ]]; then
|
|
error "SSH key $MIRROR_SSH_KEY not found — cannot reach $MIRROR"
|
|
exit 1
|
|
fi
|
|
|
|
# ── name -> template ──────────────────────────────────────────────────────────────────────────
|
|
# Matched on the <Name> element. Filenames lie often enough that trusting them would push the
|
|
# wrong container without saying so.
|
|
xml_for_container() {
|
|
local want="$1" f n
|
|
for f in "$TEMPLATES_DIR"/*.xml; do
|
|
[[ -f "$f" ]] || continue
|
|
n=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$f")
|
|
[[ "$n" == "$want" ]] && { echo "$f"; return 0; }
|
|
done
|
|
return 1
|
|
}
|
|
|
|
remote_has_container() {
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"docker inspect $(printf '%q' "$1") >/dev/null 2>&1" 2>/dev/null
|
|
}
|
|
|
|
remote_state_of() {
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"docker inspect -f '{{.State.Status}}' $(printf '%q' "$1") 2>/dev/null" 2>/dev/null
|
|
}
|
|
|
|
OK=0; FAIL=0; SKIP=0
|
|
|
|
case "$MODE" in
|
|
|
|
status)
|
|
# Written as a cache as well as printed. The assistant's fallback_state block cannot afford an
|
|
# SSH round trip per container mid-question, so it reads this file and reports its AGE — a stale
|
|
# answer stated as stale is useful, stated as current it is the exact failure this feature
|
|
# exists to prevent.
|
|
_present="" _missing=""
|
|
for c in "${COVERED[@]}"; do
|
|
if remote_has_container "$c"; then
|
|
_st=$(remote_state_of "$c")
|
|
printf ' %-28s on %s (%s)\n' "$c" "$MIRROR" "$_st"
|
|
_present+="\"$c\":\"${_st:-unknown}\","
|
|
OK=$((OK+1))
|
|
else
|
|
printf ' %-28s MISSING on %s — docker start would fail\n' "$c" "$MIRROR"
|
|
_missing+="\"$c\","
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
done
|
|
log "$ICON_FALLBACK Coverage present: $OK · missing: $FAIL"
|
|
|
|
mkdir -p "$VV_CACHE_ROOT/api" 2>/dev/null || mkdir -p /tmp/varaverk/api 2>/dev/null
|
|
_cache="${VV_CACHE_ROOT:-/tmp/varaverk}/api/fallback_presence.json"
|
|
# Written atomically — a half-written cache read mid-question would report containers as
|
|
# missing that are merely unparsed.
|
|
printf '{"present":{%s},"missing":[%s],"partner":"%s","checked":%s}\n' \
|
|
"${_present%,}" "${_missing%,}" "$MIRROR" "$(date +%s)" > "$_cache.tmp" \
|
|
&& mv -f "$_cache.tmp" "$_cache"
|
|
|
|
[[ "$FAIL" -gt 0 ]] && exit 2 || exit 0
|
|
;;
|
|
|
|
push)
|
|
# Networks first — a container whose network is absent is created and then cannot start,
|
|
# which is the failure that read as "auth 0/8, arr 0/5" during onboarding.
|
|
_nets=()
|
|
for c in "${COVERED[@]}"; do
|
|
x=$(xml_for_container "$c") || continue
|
|
net=$(sed -n 's/.*<Network>\([^<]*\)<\/Network>.*/\1/p' "$x" 2>/dev/null | head -1)
|
|
net="${net//[[:space:]]/}"
|
|
# br* is host hardware. wg* is a WireGuard-backed bridge whose meaning does NOT travel:
|
|
# recreating it on the partner as a plain bridge yields a network that exists, starts its
|
|
# containers, and routes their traffic OUTSIDE the tunnel. ChannelTube rides wg0 here.
|
|
case "$net" in
|
|
''|bridge|host|none|br[0-9]*) continue ;;
|
|
wg[0-9]*)
|
|
warn "$c uses $net — a WireGuard-backed network. NOT created on $MIRROR: a plain"
|
|
warn " bridge of the same name would route its traffic outside the tunnel. Build the"
|
|
warn " matching tunnel there first, or drop $c from coverage."
|
|
continue ;;
|
|
esac
|
|
_seen=false
|
|
for n in "${_nets[@]}"; do [[ "$n" == "$net" ]] && { _seen=true; break; }; done
|
|
[[ "$_seen" == false ]] && _nets+=("$net")
|
|
done
|
|
for net in "${_nets[@]}"; do
|
|
driver=$(timeout "${DOCKER_TIMEOUT:-30}" docker network inspect "$net" --format '{{.Driver}}' 2>/dev/null)
|
|
if [[ "$driver" != "bridge" ]]; then
|
|
warn "Network $net is '${driver:-absent}' here, not bridge — create it on $MIRROR by hand"
|
|
continue
|
|
fi
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would ensure network $net on $MIRROR"
|
|
continue
|
|
fi
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"docker network inspect $(printf '%q' "$net") >/dev/null 2>&1 \
|
|
|| docker network create --driver bridge $(printf '%q' "$net") >/dev/null" 2>/dev/null \
|
|
&& log " network $net ready on $MIRROR" \
|
|
|| warn " could not ensure network $net on $MIRROR"
|
|
done
|
|
|
|
for c in "${COVERED[@]}"; do
|
|
x=$(xml_for_container "$c") || {
|
|
warn "$c — no template in $TEMPLATES_DIR names it; skipped"
|
|
SKIP=$((SKIP+1)); continue
|
|
}
|
|
if ! deploy_container_from_xml "$x" "$MIRROR_IP" "$MIRROR_SSH_KEY"; then
|
|
error "$c — deploy failed"
|
|
FAIL=$((FAIL+1)); continue
|
|
fi
|
|
if [[ "$DRY_RUN" == true ]]; then OK=$((OK+1)); continue; fi
|
|
|
|
# Deployed containers must not run here. Stop, then re-inspect — a stop that did not take
|
|
# is the one outcome that silently duplicates a live service against shared data.
|
|
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"docker stop $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null
|
|
st=$(remote_state_of "$c")
|
|
if [[ "$st" == "running" ]]; then
|
|
error "$c is RUNNING on $MIRROR after deploy and would not stop — stop it there before continuing"
|
|
FAIL=$((FAIL+1))
|
|
else
|
|
log " $c deployed and ${st:-stopped} on $MIRROR ✅"
|
|
OK=$((OK+1))
|
|
fi
|
|
done
|
|
log "$ICON_FALLBACK Push complete — deployed $OK · failed $FAIL · skipped $SKIP"
|
|
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
|
;;
|
|
|
|
remove)
|
|
for c in "${COVERED[@]}"; do
|
|
if ! remote_has_container "$c"; then
|
|
log " $c not on $MIRROR — nothing to remove"
|
|
SKIP=$((SKIP+1)); continue
|
|
fi
|
|
# Binds are read BEFORE the container goes — once it is removed there is nothing left to
|
|
# enumerate, and a path list gathered afterwards would silently be empty.
|
|
_binds=$(timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' $(printf '%q' "$c") 2>/dev/null \
|
|
| awk -F: '{print \$1}'" 2>/dev/null)
|
|
|
|
_wipe=()
|
|
while IFS= read -r _p; do
|
|
[[ -z "$_p" ]] && continue
|
|
# Only appdata, and never an appdata root. /mnt/user/appdata as a bind would make one
|
|
# container removal delete every application on the partner.
|
|
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/.+ ]] || {
|
|
[[ "$_p" =~ ^/mnt/[^/]+/appdata[^/]*/?$ ]] && \
|
|
warn " $c binds the appdata ROOT ($_p) — refusing to delete it"
|
|
continue
|
|
}
|
|
_wipe+=("$_p")
|
|
done <<< "$_binds"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would stop and remove $c on $MIRROR"
|
|
for _p in "${_wipe[@]}"; do warn " DRY RUN — would delete appdata $_p on $MIRROR"; done
|
|
OK=$((OK+1)); continue
|
|
fi
|
|
x=$(xml_for_container "$c") && xml_name=$(basename "$x") || xml_name=""
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"docker stop $(printf '%q' "$c") >/dev/null 2>&1; \
|
|
docker rm $(printf '%q' "$c") >/dev/null 2>&1; \
|
|
${xml_name:+rm -f ${TEMPLATES_DIR}/$(printf '%q' "$xml_name");} \
|
|
! docker inspect $(printf '%q' "$c") >/dev/null 2>&1" 2>/dev/null; then
|
|
log " $c removed from $MIRROR ✅"
|
|
for _p in "${_wipe[@]}"; do
|
|
if timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
|
|
-o BatchMode=yes -o StrictHostKeyChecking=no root@"$MIRROR_IP" \
|
|
"rm -rf -- $(printf '%q' "$_p") && ! [ -e $(printf '%q' "$_p") ]" 2>/dev/null; then
|
|
log " appdata deleted on $MIRROR: $_p"
|
|
else
|
|
warn " could not delete appdata on $MIRROR: $_p"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
done
|
|
OK=$((OK+1))
|
|
else
|
|
error "$c — removal failed or it still exists on $MIRROR"
|
|
FAIL=$((FAIL+1))
|
|
fi
|
|
done
|
|
log "$ICON_FALLBACK Remove complete — removed $OK · failed $FAIL · skipped $SKIP"
|
|
[[ "$FAIL" -gt 0 ]] && exit 1 || exit 0
|
|
;;
|
|
esac
|