Report each teardown and setup step from what it did, not from whether it was attempted

Partnership/partnership_manager.sh

@@ -532,6 +532,10 @@ gather_partner_fallback_containers() {
 
 # Start this server's own parked containers after partnership ends.
 start_own_stack() {
+    # Returns non-zero if any container failed. It used to return whatever the loop's last
+    # docker start happened to produce, so a caller checking it learned nothing — and the
+    # offboard summary just printed "Step 6 — Own stack: started" either way.
+    local _rc=0
     echo ""
     echo "━━━ $ICON_START Restart Own Stack ━━━"
     if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
@@ -548,14 +552,20 @@ start_own_stack() {
             echo "$container started ✅"
         else
             warn "$container failed to start — check manually"
+            _rc=1
         fi
     done
+    return "$_rc"
 }
 
 # Remove partnership containers on this server + their appdata bind-mount paths.
 # Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
 # Safety gate: only paths matching /mnt/*/appdata* are deleted.
 cleanup_partner_containers() {
+    # Returns non-zero if any container or appdata path could not be removed. Previously the
+    # exit status was whatever the trailing while-loop produced, so "Step 5 — Local cleanup: ✅"
+    # was printed over a container that failed to remove.
+    local _rc=0
     declare -a containers=()
     gather_partner_fallback_containers containers
 
@@ -586,8 +596,12 @@ cleanup_partner_containers() {
         if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
             timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
             _PM_TRAP_STOPPED+=("$container")
-            timeout "${DOCKER_TIMEOUT:-30}" docker rm   "$container" >/dev/null 2>&1 && \
+            if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
-                echo "$container removed ✅" || warn "$container rm failed"
+                echo "$container removed ✅"
+            else
+                warn "$container rm failed"
+                _rc=1
+            fi
         else
             log "$container not found — skipping"
         fi
@@ -600,8 +614,14 @@ cleanup_partner_containers() {
             warn "  DRY RUN — would rm -rf $path"
             continue
         fi
-        rm -rf "$path" && echo "  Appdata removed: $path ✅" || warn "  Failed to remove: $path"
+        if rm -rf "$path"; then
+            echo "  Appdata removed: $path ✅"
+        else
+            warn "  Failed to remove: $path"
+            _rc=1
+        fi
     done <<< "$all_appdata_paths"
+    return "$_rc"
 }
 
 # SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
@@ -609,6 +629,10 @@ cleanup_partner_containers() {
 # Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
 # Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
 cleanup_owner_containers_on_mirror() {
+    # Returns non-zero if any remote removal failed, so the caller can report Step 7 honestly
+    # rather than from MIRROR_REACHABLE — which only says the mirror answered, not that the
+    # containers on it are gone.
+    local _rc=0
     local mirror_ip="$1"
     local owner_short
     owner_short=$(derive_short_name "$OWNER")
@@ -639,24 +663,31 @@ cleanup_owner_containers_on_mirror() {
             "docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
              | awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
 
-        timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
+        if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
             -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
             "docker stop '$container' >/dev/null 2>&1
              docker rm   '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
-            grep -q removed && \
+            grep -q removed; then
-            echo "$container removed from $MIRROR ✅" || \
+            echo "$container removed from $MIRROR ✅"
+        else
             warn "Failed to remove $container from $MIRROR"
+            _rc=1
+        fi
 
         # Delete appdata on remote after container removal
         while IFS= read -r path; do
             [[ -z "$path" ]] && continue
-            timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
+            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                 -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
-                "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
+                "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
-                echo "  Appdata removed on $MIRROR: $path ✅" || \
+                echo "  Appdata removed on $MIRROR: $path ✅"
+            else
                 warn "  Failed to remove appdata on $MIRROR: $path"
+                _rc=1
+            fi
         done <<< "$appdata_paths"
     done <<< "$container_list"
+    return "$_rc"
 }
 
 # SSH to mirror — start mirror's own parked containers.
@@ -678,18 +709,23 @@ start_mirror_own_stack() {
     fi
 
     log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
+    local _rc=0
     for container in "${mirror_own[@]}"; do
         if [[ "$DRY_RUN" == true ]]; then
             warn "DRY RUN — would start $container on $MIRROR"
             continue
         fi
-        timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
+        if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
             -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
             "docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
-            grep -q started && \
+            grep -q started; then
-            echo "$container started on $MIRROR ✅" || \
+            echo "$container started on $MIRROR ✅"
+        else
             warn "$container failed to start on $MIRROR — check manually"
+            _rc=1
+        fi
     done
+    return "$_rc"
 }
 
 # Create the mirror's Emby admin account on the owner's deployed Emby.

Partnership/partnership_offboard.sh

@@ -157,6 +157,15 @@ source "$SCRIPTS_ROOT/Plugin/$PLATFORM/Partnership/containers.sh"
 REASON="manual"
 STEP_DISABLE_RSYNC_OK=true   # both paths report it; only the mirror path re-initialised it
 TAILSCALE_REMOVED=false      # set only when remove_tailscale_device actually succeeds
+
+# Owner-path step outcomes. Every one of these was a hardcoded ✅ in the summary, or derived from
+# MIRROR_REACHABLE — which says the mirror answered a ping, not that the work on it succeeded.
+# An offboard that failed to remove a single container still reported a clean teardown.
+STEP_LOCAL_CLEANUP_OK=true
+STEP_OWN_STACK_OK=true
+STEP_REMOTE_CLEANUP_OK=true      # or "skipped" when the mirror is unreachable
+STEP_MIRROR_STACK_OK=true        # or "skipped"
+STEP_STATE_WRITE_OK=true
 FILTERED_ARGS=()
 
 for arg in "$@"; do
@@ -523,10 +532,10 @@ fi
 echo ""
 echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
 
-cleanup_partner_containers
+cleanup_partner_containers || STEP_LOCAL_CLEANUP_OK=false
 
 # ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
-start_own_stack
+start_own_stack || STEP_OWN_STACK_OK=false
 
 # ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
 echo ""
@@ -534,19 +543,24 @@ echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━
 
 if [[ "$MIRROR_REACHABLE" == true ]]; then
     # Remove auth/arr stack containers deployed during onboard (by config array)
-    cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
+    cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" || STEP_REMOTE_CLEANUP_OK=false
     # Remove fallback coverage containers (by *-owner_short naming pattern)
-    cleanup_owner_containers_on_mirror "$MIRROR_IP"
+    cleanup_owner_containers_on_mirror "$MIRROR_IP" || STEP_REMOTE_CLEANUP_OK=false
 else
     warn "$MIRROR unreachable — remote container cleanup skipped"
     warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
+    STEP_REMOTE_CLEANUP_OK=skipped
 fi
 
 # ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
 echo ""
 echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
 
-[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
+if [[ "$MIRROR_REACHABLE" == true ]]; then
+    start_mirror_own_stack "$MIRROR_IP" || STEP_MIRROR_STACK_OK=false
+else
+    STEP_MIRROR_STACK_OK=skipped
+fi
 
 # ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
 echo ""
@@ -568,9 +582,16 @@ echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
 NOW=$(date '+%Y-%m-%d %H:%M:%S')
 
 if [[ "$DRY_RUN" == false ]]; then
-    write_state_file "$LOCAL_STATE_FILE" \
+    # Checked, because this is the record every other host and every later --check reads. A
-        "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
+    # failed write here leaves both sides believing the partnership is still active while the
-    echo "Local state: INACTIVE ✅"
+    # summary says INACTIVE — the one line in the teardown that must not be assumed.
+    if write_state_file "$LOCAL_STATE_FILE" \
+        "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"; then
+        echo "Local state: INACTIVE ✅"
+    else
+        error "Failed to write local state file — $MIRROR may still look ACTIVE here"
+        STEP_STATE_WRITE_OK=false
+    fi
     add_to_blocklist "$MIRROR" "$REASON"
     [[ "$MIRROR_REACHABLE" == true ]] && \
         push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
@@ -637,12 +658,12 @@ echo "  Step 1  — Stop rsync:     $(_ok "$STEP_STOP_OK")"
 echo "  Step 2  — Final sync:     $(_ok "$STEP_SYNC_OK")"
 echo "  Step 3  — WebUI failures: $WEBUI_FAILURES"
 echo "  Step 4  — Sync gates:     $(_ok "$STEP_DISABLE_RSYNC_OK") (${_VV_SYNC_GATES[*]} → false)"
-echo "  Step 5  — Local cleanup:  ✅"
+echo "  Step 5  — Local cleanup:  $(_ok "$STEP_LOCAL_CLEANUP_OK")"
-echo "  Step 6  — Own stack:      started"
+echo "  Step 6  — Own stack:      $( [[ "$STEP_OWN_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️  check warnings above" )"
-echo "  Step 7  — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
+echo "  Step 7  — Remote cleanup: $( [[ "$STEP_REMOTE_CLEANUP_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$STEP_REMOTE_CLEANUP_OK" )"
-echo "  Step 8  — Mirror stack:   $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
+echo "  Step 8  — Mirror stack:   $( [[ "$STEP_MIRROR_STACK_OK" == skipped ]] && echo "skipped (unreachable)" || { [[ "$STEP_MIRROR_STACK_OK" == true ]] && echo "started ✅" || echo "⚠️  check warnings above"; } )"
 echo "  Step 9  — Keys revoked:   $(_revoke_status)"
-echo "  Step 10 — State:          INACTIVE ✅"
+echo "  Step 10 — State:          $( [[ "$STEP_STATE_WRITE_OK" == true ]] && echo "INACTIVE ✅" || echo "⚠️  WRITE FAILED — still looks ACTIVE here" )"
 echo ""
 echo "  Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
 if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then

Partnership/partnership_onboard.sh

@@ -448,6 +448,7 @@ echo ""
 
 STEP_SSH_OK=false
 STEP_NETWORK_OK=false
+LOCAL_SETUP_OK=true      # partnership_manager --local-only; the summary claimed done ✅ regardless
 PHASE1_NET_OK=false      # Phase 1 only — network created on the mirror before any deploy
 PHASE1_CACHE_OK=false    # Phase 1 only — our conf pushed into the mirror's RAM cache
 STEP_STOP_AUTH_OK=true
@@ -533,15 +534,17 @@ if [[ "$PHASE1_ONLY" == true ]]; then
         # UI will show "key ready, install manually" state via HOST2_KEY_READY flag.
         echo ""
         echo "━━━ Phase 1 — HOST1 Local Setup (SSH pending) ━━━"
-        bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
+        if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
+            LOCAL_SETUP_OK=false
             warn "Local setup had issues — check partnership_manager.sh output above"
+        fi
 
         END=$(date +%s)
         echo ""
         echo "━━━━━ $ICON_SUMMARY PHASE 1 — SSH PENDING ━━━━━"
         echo "  SSH keys:    key generated ✅ — NOT yet installed on $MIRROR ⚠"
         echo "  Conf push:   skipped (needs SSH access to $MIRROR)"
-        echo "  HOST1 setup: done ✅"
+        echo "  HOST1 setup: $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️  had issues — see above" )"
         echo "  Duration:    $(format_duration $(( END - START )))"
         echo ""
         echo "  ACTION NEEDED: install the key on $MIRROR:"
@@ -634,8 +637,10 @@ if [[ "$PHASE1_ONLY" == true ]]; then
     # HOST1 local setup — runs immediately without needing HOST2
     echo ""
     echo "━━━ Phase 1 — HOST1 Local Setup ━━━"
-    bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}" || \
+    if ! bash "$SCRIPT_DIR/partnership_manager.sh" --onboard --local-only "${EXTRA_FLAGS[@]}"; then
+        LOCAL_SETUP_OK=false
         warn "Local setup had issues — check partnership_manager.sh output above"
+    fi
 
     [[ "$DRY_RUN" == false ]] && write_onboard_phase "$MIRROR_ID" 1
 
@@ -646,7 +651,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then
     echo "  Conf push:    $( [[ "$CONF_PUSH_OK" == true ]] && echo "done ✅" || echo "⚠ manual needed" )"
     echo "  Network:      $( [[ "$PHASE1_NET_OK" == true ]] && echo "ready on $MIRROR ✅" || echo "⚠ Step 1b will retry" )"
     echo "  Conf cache:   $( [[ "$PHASE1_CACHE_OK" == true ]] && echo "pushed to $MIRROR ✅" || echo "⚠ not cached" )"
-    echo "  HOST1 setup:  done ✅"
+    echo "  HOST1 setup:  $( [[ "$LOCAL_SETUP_OK" == true ]] && echo "done ✅" || echo "⚠️  had issues — see above" )"
     echo "  Duration:     $(format_duration $(( END - START )))"
     echo ""
     echo "  HOST1 is fully set up. HOST2 ($MIRROR) can now install the Varaverk plugin."

Plugin/unraid/Partnership/containers.sh

@@ -507,6 +507,7 @@ ensure_stack_networks_on_remote() {
 # ==============================================================================================
 cleanup_deployed_stack_on_remote() {
     local remote_ip="$1" ssh_key="$2"
+    local _rc=0
     local -a xml_names=()
     [[ ${#PARTNERSHIP_AUTH_STACK[@]}     -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
     [[ ${#PARTNERSHIP_ARR_STACK[@]}      -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
@@ -552,13 +553,20 @@ cleanup_deployed_stack_on_remote() {
 
         while IFS= read -r path; do
             [[ -z "$path" ]] && continue
-            timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
+            if timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
                 -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
-                "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
+                "rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed; then
-                echo "  Appdata removed on $MIRROR: $path ✅" || \
+                echo "  Appdata removed on $MIRROR: $path ✅"
+            else
                 warn "  Failed to remove appdata on $MIRROR: $path"
+                _rc=1
+            fi
         done <<< "$appdata_paths"
     done
+    # Only appdata failures are counted. The container branch above cannot tell "removal failed"
+    # from "already gone" — both produce no `removed` echo — and an offboard re-run on a
+    # half-finished teardown is a normal case, so treating that as failure would cry wolf.
+    return "$_rc"
 }
 
 # ==============================================================================================

Plugin/unraid/pages/partnership.php

@@ -182,7 +182,13 @@ function _vvPtRun(id, extraArgs) {
     if (!r.ok) throw new Error('HTTP ' + r.status);
     return r.text();
   }).then(text => {
-    if (!text.trim()) return {ok: true, _empty_response: true};
+    // An empty body is never success. api/run.php echoes JSON on every path it can reach, so
+    // nothing arriving means the request died before it: Unraid's CSRF guard exits with an empty
+    // body on a token mismatch, nginx returns an empty 200 for a bodyless POST, and a PHP fatal
+    // produces the same. Returning {ok:true} for that made a killed request and a launched job
+    // indistinguishable — press Onboard, get no error, and nothing has run. vvApiKey below has
+    // always thrown on this; the action path is what disagreed.
+    if (!text.trim()) throw new Error('Empty response — request rejected before it reached run.php');
     return JSON.parse(text);
   });
 }