From 5bdf3bff604d4db6155ddda7d83e185156cd14eb Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 16 Aug 2026 21:46:26 -0400 Subject: [PATCH] Report each teardown and setup step from what it did, not from whether it was attempted --- Partnership/partnership_manager.sh | 60 ++++++++++++++++++++----- Partnership/partnership_offboard.sh | 47 +++++++++++++------ Partnership/partnership_onboard.sh | 13 ++++-- Plugin/unraid/Partnership/containers.sh | 14 ++++-- Plugin/unraid/pages/partnership.php | 8 +++- 5 files changed, 109 insertions(+), 33 deletions(-) diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index ac534c7..f5a13fc 100755 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -532,6 +532,10 @@ gather_partner_fallback_containers() { # Start this server's own parked containers after partnership ends. start_own_stack() { + # Returns non-zero if any container failed. It used to return whatever the loop's last + # docker start happened to produce, so a caller checking it learned nothing — and the + # offboard summary just printed "Step 6 — Own stack: started" either way. + local _rc=0 echo "" echo "━━━ $ICON_START Restart Own Stack ━━━" if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then @@ -548,14 +552,20 @@ start_own_stack() { echo "$container started ✅" else warn "$container failed to start — check manually" + _rc=1 fi done + return "$_rc" } # Remove partnership containers on this server + their appdata bind-mount paths. # Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers. # Safety gate: only paths matching /mnt/*/appdata* are deleted. cleanup_partner_containers() { + # Returns non-zero if any container or appdata path could not be removed. Previously the + # exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅" + # was printed over a container that failed to remove. + local _rc=0 declare -a containers=() gather_partner_fallback_containers containers @@ -586,8 +596,12 @@ cleanup_partner_containers() { if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true _PM_TRAP_STOPPED+=("$container") - timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \ - echo "$container removed ✅" || warn "$container rm failed" + if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then + echo "$container removed ✅" + else + warn "$container rm failed" + _rc=1 + fi else log "$container not found — skipping" fi @@ -600,8 +614,14 @@ cleanup_partner_containers() { warn " DRY RUN — would rm -rf $path" continue fi - rm -rf "$path" && echo " Appdata removed: $path ✅" || warn " Failed to remove: $path" + if rm -rf "$path"; then + echo " Appdata removed: $path ✅" + else + warn " Failed to remove: $path" + _rc=1 + fi done <<< "$all_appdata_paths" + return "$_rc" } # SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers) @@ -609,6 +629,10 @@ cleanup_partner_containers() { # Appdata paths collected via SSH docker inspect before removal, then deleted via SSH. # Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote. cleanup_owner_containers_on_mirror() { + # Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly + # rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the + # containers on it are gone. + local _rc=0 local mirror_ip="$1" local owner_short owner_short=$(derive_short_name "$OWNER") @@ -639,24 +663,31 @@ cleanup_owner_containers_on_mirror() { "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \ | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null) - timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker stop '$container' >/dev/null 2>&1 docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \ - grep -q removed && \ - echo "$container removed from $MIRROR ✅" || \ + grep -q removed; then + echo "$container removed from $MIRROR ✅" + else warn "Failed to remove $container from $MIRROR" + _rc=1 + fi # Delete appdata on remote after container removal while IFS= read -r path; do [[ -z "$path" ]] && continue - timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ - "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \ - echo " Appdata removed on $MIRROR: $path ✅" || \ + "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then + echo " Appdata removed on $MIRROR: $path ✅" + else warn " Failed to remove appdata on $MIRROR: $path" + _rc=1 + fi done <<< "$appdata_paths" done <<< "$container_list" + return "$_rc" } # SSH to mirror — start mirror's own parked containers. @@ -678,18 +709,23 @@ start_mirror_own_stack() { fi log "Restarting own stack on $MIRROR: ${mirror_own[*]}" + local _rc=0 for container in "${mirror_own[@]}"; do if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would start $container on $MIRROR" continue fi - timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ + if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \ "docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \ - grep -q started && \ - echo "$container started on $MIRROR ✅" || \ + grep -q started; then + echo "$container started on $MIRROR ✅" + else warn "$container failed to start on $MIRROR — check manually" + _rc=1 + fi done + return "$_rc" } # Create the mirror's Emby admin account on the owner's deployed Emby. diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh index d835907..62cf37e 100755 --- a/Partnership/partnership_offboard.sh +++ b/Partnership/partnership_offboard.sh @@ -157,6 +157,15 @@ source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh" REASON="manual" STEP_DISABLE_RSYNC_OK=true # both paths report it; only the mirror path re-initialised it TAILSCALE_REMOVED=false # set only when remove_tailscale_device actually succeeds + +# Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from +# MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded. +# An offboard that failed to remove a single container still reported a clean teardown. +STEP_LOCAL_CLEANUP_OK=true +STEP_OWN_STACK_OK=true +STEP_REMOTE_CLEANUP_OK=true # or "skipped" when the mirror is unreachable +STEP_MIRROR_STACK_OK=true # or "skipped" +STEP_STATE_WRITE_OK=true FILTERED_ARGS=() for arg in "$@"; do @@ -523,10 +532,10 @@ fi echo "" echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━" -cleanup_partner_containers +cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false # ── Step 6: Restart own stack ───────────────────────────────────────────────────────────────── -start_own_stack +start_own_stack || STEP_OWN_STACK_OK=false # ── Step 7: Remote container cleanup ────────────────────────────────────────────────────────── echo "" @@ -534,19 +543,24 @@ echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━ if [[ "$MIRROR_REACHABLE" == true ]]; then # Remove auth/arr stack containers deployed during onboard (by config array) - cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" + cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false # Remove fallback coverage containers (by *-owner_short naming pattern) - cleanup_owner_containers_on_mirror "$MIRROR_IP" + cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false else warn "$MIRROR unreachable — remote container cleanup skipped" warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually" + STEP_REMOTE_CLEANUP_OK=skipped fi # ── Step 8: Restart mirror's own stack ──────────────────────────────────────────────────────── echo "" echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━" -[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP" +if [[ "$MIRROR_REACHABLE" == true ]]; then + start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false +else + STEP_MIRROR_STACK_OK=skipped +fi # ── Step 9: Revocation (Emby + SSH) ────────────────────────────────────────────────────────── echo "" @@ -568,9 +582,16 @@ echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') if [[ "$DRY_RUN" == false ]]; then - write_state_file "$LOCAL_STATE_FILE" \ - "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" - echo "Local state: INACTIVE ✅" + # Checked, because this is the record every other host and every later --check reads. A + # failed write here leaves both sides believing the partnership is still active while the + # summary says INACTIVE — the one line in the teardown that must not be assumed. + if write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then + echo "Local state: INACTIVE ✅" + else + error "Failed to write local state file — $MIRROR may still look ACTIVE here" + STEP_STATE_WRITE_OK=false + fi add_to_blocklist "$MIRROR" "$REASON" [[ "$MIRROR_REACHABLE" == true ]] && \ push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" @@ -637,12 +658,12 @@ echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")" echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")" echo " Step 3 — WebUI failures: $WEBUI_FAILURES" echo " Step 4 — Sync gates: $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)" -echo " Step 5 — Local cleanup: ✅" -echo " Step 6 — Own stack: started" -echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )" -echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )" +echo " Step 5 — Local cleanup: $(_ok "$STEP_LOCAL_CLEANUP_OK")" +echo " Step 6 — Own stack: $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above" )" +echo " Step 7 — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )" +echo " Step 8 — Mirror stack: $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️ check warnings above"; } )" echo " Step 9 — Keys revoked: $(_revoke_status)" -echo " Step 10 — State: INACTIVE ✅" +echo " Step 10 — State: $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️ WRITE FAILED — still looks ACTIVE here" )" echo "" echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 348e3de..2de804c 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -448,6 +448,7 @@ echo "" STEP_SSH_OK=false STEP_NETWORK_OK=false +LOCAL_SETUP_OK=true # partnership_manager --local-only; the summary claimed done ✅ regardless PHASE1_NET_OK=false # Phase 1 only — network created on the mirror before any deploy PHASE1_CACHE_OK=false # Phase 1 only — our conf pushed into the mirror's RAM cache STEP_STOP_AUTH_OK=true @@ -533,15 +534,17 @@ if [[ "$PHASE1_ONLY" == true ]]; then # UI will show "key ready, install manually" state via HOST2_KEY_READY flag. echo "" echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━" - bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \ + if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then + LOCAL_SETUP_OK=false warn "Local setup had issues — check partnership_manager.sh output above" + fi END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━" echo " SSH keys: key generated ✅ — NOT yet installed on $MIRROR ⚠" echo " Conf push: skipped (needs SSH access to $MIRROR)" - echo " HOST1 setup: done ✅" + echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )" echo " Duration: $(format_duration $(( END - START )))" echo "" echo " ACTION NEEDED: install the key on $MIRROR:" @@ -634,8 +637,10 @@ if [[ "$PHASE1_ONLY" == true ]]; then # HOST1 local setup — runs immediately without needing HOST2 echo "" echo "━━━ Phase 1 — HOST1 Local Setup ━━━" - bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \ + if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then + LOCAL_SETUP_OK=false warn "Local setup had issues — check partnership_manager.sh output above" + fi [[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1 @@ -646,7 +651,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then echo " Conf push: $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )" echo " Network: $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )" echo " Conf cache: $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )" - echo " HOST1 setup: done ✅" + echo " HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️ had issues — see above" )" echo " Duration: $(format_duration $(( END - START )))" echo "" echo " HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin." diff --git a/Plugin/unraid/Partnership/containers.sh b/Plugin/unraid/Partnership/containers.sh index caac801..6c9aaff 100755 --- a/Plugin/unraid/Partnership/containers.sh +++ b/Plugin/unraid/Partnership/containers.sh @@ -507,6 +507,7 @@ ensure_stack_networks_on_remote() { # ============================================================================================== cleanup_deployed_stack_on_remote() { local remote_ip="$1" ssh_key="$2" + local _rc=0 local -a xml_names=() [[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}") [[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}") @@ -552,13 +553,20 @@ cleanup_deployed_stack_on_remote() { while IFS= read -r path; do [[ -z "$path" ]] && continue - timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ + if timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \ -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \ - "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \ - echo " Appdata removed on $MIRROR: $path ✅" || \ + "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then + echo " Appdata removed on $MIRROR: $path ✅" + else warn " Failed to remove appdata on $MIRROR: $path" + _rc=1 + fi done <<< "$appdata_paths" done + # Only appdata failures are counted. The container branch above cannot tell "removal failed" + # from "already gone" — both produce no `removed` echo — and an offboard re-run on a + # half-finished teardown is a normal case, so treating that as failure would cry wolf. + return "$_rc" } # ============================================================================================== diff --git a/Plugin/unraid/pages/partnership.php b/Plugin/unraid/pages/partnership.php index beba33e..aebb858 100644 --- a/Plugin/unraid/pages/partnership.php +++ b/Plugin/unraid/pages/partnership.php @@ -182,7 +182,13 @@ function _vvPtRun(id, extraArgs) { if (!r.ok) throw new Error('HTTP ' + r.status); return r.text(); }).then(text => { - if (!text.trim()) return {ok: true, _empty_response: true}; + // An empty body is never success. api/run.php echoes JSON on every path it can reach, so + // nothing arriving means the request died before it: Unraid's CSRF guard exits with an empty + // body on a token mismatch, nginx returns an empty 200 for a bodyless POST, and a PHP fatal + // produces the same. Returning {ok:true} for that made a killed request and a launched job + // indistinguishable — press Onboard, get no error, and nothing has run. vvApiKey below has + // always thrown on this; the action path is what disagreed. + if (!text.trim()) throw new Error('Empty response — request rejected before it reached run.php'); return JSON.parse(text); }); }