FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Report each teardown and setup step from what it did, not from whether it was attempted

Browse Source
This commit is contained in:
Gmer4Lfe
2026-08-16 21:46:26 -04:00
parent 65516ea3ac
commit 5bdf3bff60
5 changed files with 109 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Partnership/partnership_manager.sh
+48 -12
View File
@@ -532,6 +532,10 @@ gather_partner_fallback_containers() {
# Start this server's own parked containers after partnership ends.
start_own_stack() {
# Returns non-zero if any container failed. It used to return whatever the loop's last
# docker start happened to produce, so a caller checking it learned nothing — and the
# offboard summary just printed "Step 6 — Own stack: started" either way.
local _rc=0
echo ""
echo "━━━ $ICON_START Restart Own Stack ━━━"
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
@@ -548,14 +552,20 @@ start_own_stack() {
echo "$container started ✅"
else
warn "$container failed to start — check manually"
_rc=1
fi
done
return "$_rc"
}
# Remove partnership containers on this server + their appdata bind-mount paths.
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
cleanup_partner_containers() {
# Returns non-zero if any container or appdata path could not be removed. Previously the
# exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅"
# was printed over a container that failed to remove.
local _rc=0
declare -a containers=()
gather_partner_fallback_containers containers
@@ -586,8 +596,12 @@ cleanup_partner_containers() {
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
_PM_TRAP_STOPPED+=("$container")
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
echo "$container removed ✅" || warn "$container rm failed"
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
echo "$container removed ✅"
else
warn "$container rm failed"
_rc=1
fi
else
log "$container not found — skipping"
fi
@@ -600,8 +614,14 @@ cleanup_partner_containers() {
warn " DRY RUN — would rm -rf $path"
continue
fi
rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path"
if rm -rf "$path"; then
echo " Appdata removed: $path ✅"
else
warn " Failed to remove: $path"
_rc=1
fi
done <<< "$all_appdata_paths"
return "$_rc"
}
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
@@ -609,6 +629,10 @@ cleanup_partner_containers() {
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
cleanup_owner_containers_on_mirror() {
# Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly
# rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the
# containers on it are gone.
local _rc=0
local mirror_ip="$1"
local owner_short
owner_short=$(derive_short_name "$OWNER")
@@ -639,24 +663,31 @@ cleanup_owner_containers_on_mirror() {
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker stop '$container' >/dev/null 2>&1
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
grep -q removed && \
echo "$container removed from $MIRROR ✅" || \
grep -q removed; then
echo "$container removed from $MIRROR ✅"
else
warn "Failed to remove $container from $MIRROR"
_rc=1
fi
# Delete appdata on remote after container removal
while IFS= read -r path; do
[[ -z "$path" ]] && continue
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
echo " Appdata removed on $MIRROR: $path ✅" || \
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
echo " Appdata removed on $MIRROR: $path ✅"
else
warn " Failed to remove appdata on $MIRROR: $path"
_rc=1
fi
done <<< "$appdata_paths"
done <<< "$container_list"
return "$_rc"
}
# SSH to mirror — start mirror's own parked containers.
@@ -678,18 +709,23 @@ start_mirror_own_stack() {
fi
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
local _rc=0
for container in "${mirror_own[@]}"; do
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would start $container on $MIRROR"
continue
fi
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
grep -q started && \
echo "$container started on $MIRROR ✅" || \
grep -q started; then
echo "$container started on $MIRROR ✅"
else
warn "$container failed to start on $MIRROR — check manually"
_rc=1
fi
done
return "$_rc"
}
# Create the mirror's Emby admin account on the owner's deployed Emby.
Partnership/partnership_offboard.sh
+34 -13
View File
@@ -157,6 +157,15 @@ source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
REASON="manual"
STEP_DISABLE_RSYNC_OK=true # both paths report it; only the mirror path re-initialised it
TAILSCALE_REMOVED=false # set only when remove_tailscale_device actually succeeds
# Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from
# MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded.
# An offboard that failed to remove a single container still reported a clean teardown.
STEP_LOCAL_CLEANUP_OK=true
STEP_OWN_STACK_OK=true
STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable
STEP_MIRROR_STACK_OK=true # or "skipped"
STEP_STATE_WRITE_OK=true
FILTERED_ARGS=()
for arg in "$@"; do
@@ -523,10 +532,10 @@ fi
echo ""
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
cleanup_partner_containers
cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
start_own_stack
start_own_stack || STEP_OWN_STACK_OK=false
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
echo ""
@@ -534,19 +543,24 @@ echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━
if [[ "$MIRROR_REACHABLE" == true ]]; then
# Remove auth/arr stack containers deployed during onboard (by config array)
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false
# Remove fallback coverage containers (by *-owner_short naming pattern)
cleanup_owner_containers_on_mirror "$MIRROR_IP"
cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false
else
warn "$MIRROR unreachable — remote container cleanup skipped"
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
STEP_REMOTE_CLEANUP_OK=skipped
fi
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
if [[ "$MIRROR_REACHABLE" == true ]]; then
start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false
else
STEP_MIRROR_STACK_OK=skipped
fi
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
echo ""
@@ -568,9 +582,16 @@ echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
NOW=$(date '+%Y-%m-%d %H:%M:%S')
if [[ "$DRY_RUN" == false ]]; then
write_state_file "$LOCAL_STATE_FILE" \
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
echo "Local state: INACTIVE ✅"
# Checked, because this is the record every other host and every later --check reads. A
# failed write here leaves both sides believing the partnership is still active while the
# summary says INACTIVE — the one line in the teardown that must not be assumed.
if write_state_file "$LOCAL_STATE_FILE" \
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then
echo "Local state: INACTIVE ✅"
else
error "Failed to write local state file — $MIRROR may still look ACTIVE here"
STEP_STATE_WRITE_OK=false
fi
add_to_blocklist "$MIRROR" "$REASON"
[[ "$MIRROR_REACHABLE" == true ]] && \
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
@@ -637,12 +658,12 @@ echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
echo " Step 4 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
echo " Step 5 — Local cleanup: ✅"
echo " Step 6 — Own stack: started"
echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")"
echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )"
echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )"
echo " Step 9 — Keys revoked: $(_revoke_status)"
echo " Step 10 — State: INACTIVE ✅"
echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )"
echo ""
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
Partnership/partnership_onboard.sh
+9 -4
View File
@@ -448,6 +448,7 @@ echo ""
STEP_SSH_OK=false
STEP_NETWORK_OK=false
LOCAL_SETUP_OK=true # partnership_manager --local-only; the summary claimed done ✅ regardless
PHASE1_NET_OK=false # Phase 1 only — network created on the mirror before any deploy
PHASE1_CACHE_OK=false # Phase 1 only — our conf pushed into the mirror's RAM cache
STEP_STOP_AUTH_OK=true
@@ -533,15 +534,17 @@ if [[ "$PHASE1_ONLY" == true ]]; then
# UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
echo ""
echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
LOCAL_SETUP_OK=false
warn "Local setup had issues — check partnership_manager.sh output above"
fi
END=$(date +%s)
echo ""
echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠"
echo " Conf push: skipped (needs SSH access to $MIRROR)"
echo " HOST1 setup: done ✅"
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
echo " Duration: $(format_duration $(( END - START )))"
echo ""
echo " ACTION NEEDED: install the key on $MIRROR:"
@@ -634,8 +637,10 @@ if [[ "$PHASE1_ONLY" == true ]]; then
# HOST1 local setup — runs immediately without needing HOST2
echo ""
echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
LOCAL_SETUP_OK=false
warn "Local setup had issues — check partnership_manager.sh output above"
fi
[[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
@@ -646,7 +651,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then
echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
echo " Network: $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )"
echo " Conf cache: $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )"
echo " HOST1 setup: done ✅"
echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )"
echo " Duration: $(format_duration $(( END - START )))"
echo ""
echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."